From c2e91fcb7517d79257e7185e56a763f3f10c8024 Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 7 Nov 2019 18:12:26 +0800 Subject: [PATCH] Adjust code structure --- .gitignore | 11 +- CMakeLists.txt | 290 +- deps/CMakeLists.txt | 1 - deps/iconv/CMakeLists.txt | 3 +- deps/pthread/CMakeLists.txt | 5 +- deps/regex/CMakeLists.txt | 5 +- deps/zlib-1.2.11/CMakeLists.txt | 5 +- deps/zlib-1.2.11/inc/zconf.h | 16 +- deps/zlib-1.2.11/inc/zlib.h | 4 +- deps/zlib-1.2.11/src/adler32.c | 12 +- deps/zlib-1.2.11/src/compress.c | 2 +- deps/zlib-1.2.11/src/crc32.c | 52 +- deps/zlib-1.2.11/src/deflate.c | 12 +- deps/zlib-1.2.11/src/deflate.h | 2 +- deps/zlib-1.2.11/src/gzclose.c | 2 +- deps/zlib-1.2.11/src/gzlib.c | 2 +- deps/zlib-1.2.11/src/gzread.c | 2 +- deps/zlib-1.2.11/src/gzwrite.c | 2 +- deps/zlib-1.2.11/src/infback.c | 6 +- deps/zlib-1.2.11/src/inffast.c | 18 +- deps/zlib-1.2.11/src/inflate.c | 20 +- deps/zlib-1.2.11/src/inflate.h | 6 +- deps/zlib-1.2.11/src/inftrees.c | 2 +- deps/zlib-1.2.11/src/trees.c | 6 +- deps/zlib-1.2.11/src/uncompr.c | 2 +- deps/zlib-1.2.11/src/zconf.h | 14 +- deps/zlib-1.2.11/src/zlib.h | 4 +- deps/zlib-1.2.11/src/zutil.c | 4 +- deps/zlib-1.2.11/src/zutil.h | 6 +- packaging/cfg/taos.cfg | 4 +- packaging/deb/makedeb.sh | 4 +- packaging/deb/taosd | 0 packaging/release.sh | 2 +- packaging/rpm/taosd | 0 packaging/rpm/tdengine.spec | 2 +- packaging/tools/get_version.sh | 3 +- packaging/tools/make_install.sh | 2 +- packaging/tools/makepkg.sh | 2 +- src/CMakeLists.txt | 3 - src/client/CMakeLists.txt | 47 +- src/client/inc/tscJoinProcess.h | 155 + src/client/inc/tscSecondaryMerge.h | 103 +- src/client/inc/tscUtil.h | 147 +- src/client/inc/tsclient.h | 316 +- .../jni/com_taosdata_jdbc_TSDBJNIConnector.h | 0 src/client/src/TSDBJNIConnector.c | 3 +- src/client/src/tscAst.c | 480 +- src/client/src/tscAsync.c | 73 +- src/client/src/tscCache.c | 5 + src/client/src/tscFunctionImpl.c | 4867 ++++++++--------- src/client/src/tscJoinProcess.c | 1496 +++++ src/client/src/tscLocal.c | 96 +- src/client/src/tscParseInsert.c | 840 +-- src/client/src/tscPrepare.c | 599 ++ src/client/src/tscSQLParser.c | 3546 ++++++++---- src/client/src/tscSQLParserImpl.c | 91 +- src/client/src/tscSchemaUtil.c | 65 +- src/client/src/tscSecondaryMerge.c | 311 +- src/client/src/tscServer.c | 1944 +++++-- src/client/src/tscSql.c | 560 +- src/client/src/tscStream.c | 81 +- src/client/src/tscSub.c | 2 +- src/client/src/tscSyntaxtreefunction.c | 330 +- src/client/src/tscSystem.c | 39 +- src/client/src/tscUtil.c | 1010 +++- .../java/com/taosdata/jdbc/TSDBError.java | 2 +- .../src/test/java/TestPreparedStatement.java | 29 + .../test/java/TestTSDBDatabaseMetaData.java | 29 + src/inc/buildInfo.h | 0 src/inc/lz4.h | 11 +- src/inc/sdb.h | 34 + src/inc/sql.y | 85 +- src/inc/taos.h | 55 +- src/inc/taosmsg.h | 201 +- src/inc/tast.h | 64 +- src/inc/textbuffer.h | 6 +- src/inc/tglobalcfg.h | 46 +- src/inc/tidpool.h | 4 + src/inc/trpc.h | 3 +- src/inc/tschemautil.h | 7 +- src/inc/tsdb.h | 154 +- src/inc/tskiplist.h | 81 +- src/inc/tsql.h | 40 +- src/inc/tsqldef.h | 290 +- src/inc/tsqlfunction.h | 260 +- src/inc/tstatus.h | 2 + src/inc/tstoken.h | 36 +- src/inc/tstrbuild.h | 52 + src/inc/tsystem.h | 2 + src/inc/ttimer.h | 2 + src/inc/ttypes.h | 4 +- src/inc/tutil.h | 59 +- src/kit/CMakeLists.txt | 1 - src/kit/shell/CMakeLists.txt | 35 +- src/kit/shell/inc/shell.h | 2 + src/kit/shell/src/shellEngine.c | 156 +- src/kit/shell/src/shellLinux.c | 4 +- src/kit/shell/src/shellMain.c | 3 + src/kit/shell/src/shellWindows.c | 7 +- src/kit/taosdemo/CMakeLists.txt | 9 +- src/kit/taosdemo/taosdemo.c | 24 +- src/kit/taosdump/CMakeLists.txt | 9 +- src/kit/taosdump/taosdump.c | 341 +- src/modules/CMakeLists.txt | 1 - src/modules/http/CMakeLists.txt | 13 +- src/modules/http/inc/httpHandle.h | 26 +- src/modules/http/src/cJSON.c | 2 +- src/modules/http/src/httpHandle.c | 26 +- src/modules/http/src/httpJson.c | 2 +- src/modules/http/src/httpResp.c | 10 +- src/modules/http/src/httpServer.c | 231 +- src/modules/http/src/httpSql.c | 3 + src/modules/http/src/httpSystem.c | 16 +- src/modules/http/src/tgHandle.c | 2 +- src/modules/monitor/CMakeLists.txt | 8 +- src/modules/monitor/src/monitorSystem.c | 90 + src/os/CMakeLists.txt | 1 - src/os/darwin/CMakeLists.txt | 6 +- src/os/darwin/inc/os.h | 5 + src/os/linux/CMakeLists.txt | 6 +- src/os/linux/inc/os.h | 51 +- src/os/linux/src/tsystem.c | 4 +- src/os/windows/CMakeLists.txt | 7 +- src/os/windows/inc/os.h | 8 + src/os/windows/src/twindows.c | 12 + src/os/windows/src/twintcpclient.c | 10 +- src/os/windows/src/twintcpserver.c | 8 +- src/rpc/CMakeLists.txt | 13 +- src/rpc/src/trpc.c | 42 +- src/rpc/src/tstring.c | 24 +- src/rpc/src/ttcpclient.c | 30 +- src/rpc/src/ttcpserver.c | 3 + src/rpc/src/tudp.c | 64 +- src/sdb/CMakeLists.txt | 17 +- src/sdb/src/sdbEngine.c | 331 +- src/sdb/src/sdbstr.c | 25 + src/system/CMakeLists.txt | 30 +- src/system/detail/CMakeLists.txt | 42 + src/system/{ => detail}/inc/dnodeSystem.h | 23 +- src/system/{ => detail}/inc/mgmt.h | 91 +- src/system/detail/inc/mgmtBalance.h | 83 + src/system/{ => detail}/inc/mgmtProfile.h | 0 src/system/{ => detail}/inc/mgmtSystem.h | 22 +- src/system/detail/inc/mgmtUtil.h | 40 + src/system/{ => detail}/inc/vnode.h | 123 +- src/system/{ => detail}/inc/vnodeCache.h | 6 +- .../{ => detail}/inc/vnodeDataFilterFunc.h | 0 src/system/{ => detail}/inc/vnodeFile.h | 25 +- src/system/{ => detail}/inc/vnodeMgmt.h | 12 +- src/system/detail/inc/vnodePeer.h | 108 + src/system/{ => detail}/inc/vnodeQueryImpl.h | 81 +- src/system/{ => detail}/inc/vnodeRead.h | 128 +- src/system/{ => detail}/inc/vnodeShell.h | 0 src/system/{ => detail}/inc/vnodeStore.h | 7 + src/system/{ => detail}/inc/vnodeSystem.h | 14 + src/system/{ => detail}/inc/vnodeTagMgmt.h | 2 +- src/system/{ => detail}/inc/vnodeUtil.h | 4 + src/system/{ => detail}/src/dnodeMgmt.c | 233 +- src/system/{ => detail}/src/dnodeService.c | 3 + src/system/{ => detail}/src/dnodeSystem.c | 117 +- src/system/{ => detail}/src/mgmtAcct.c | 14 +- src/system/{ => detail}/src/mgmtConn.c | 1 + src/system/{ => detail}/src/mgmtDb.c | 151 +- src/system/detail/src/mgmtDnode.c | 391 ++ src/system/detail/src/mgmtDnodeInt.c | 484 ++ src/system/{ => detail}/src/mgmtMeter.c | 624 +-- src/system/{ => detail}/src/mgmtProfile.c | 5 +- src/system/{ => detail}/src/mgmtShell.c | 463 +- src/system/detail/src/mgmtSupertableQuery.c | 842 +++ src/system/detail/src/mgmtSystem.c | 162 + src/system/{ => detail}/src/mgmtUser.c | 32 +- src/system/{ => detail}/src/mgmtUtil.c | 45 + src/system/{ => detail}/src/mgmtVgroup.c | 162 +- src/system/{ => detail}/src/vnodeCache.c | 183 +- src/system/{ => detail}/src/vnodeCommit.c | 4 +- src/system/{ => detail}/src/vnodeFile.c | 282 +- src/system/detail/src/vnodeFileUtil.c | 241 + src/system/detail/src/vnodeFilterFunc.c | 562 ++ src/system/{ => detail}/src/vnodeImport.c | 40 +- src/system/{ => detail}/src/vnodeMeter.c | 66 +- src/system/{ => detail}/src/vnodeQueryImpl.c | 2269 +++++--- .../{ => detail}/src/vnodeQueryProcess.c | 409 +- src/system/{ => detail}/src/vnodeRead.c | 199 +- src/system/{ => detail}/src/vnodeShell.c | 87 +- src/system/{ => detail}/src/vnodeStore.c | 66 +- src/system/{ => detail}/src/vnodeStream.c | 24 +- src/system/{ => detail}/src/vnodeSystem.c | 59 +- .../src/vnodeTagMgmt.c} | 95 +- src/system/{ => detail}/src/vnodeUtil.c | 198 +- src/system/lite/CMakeLists.txt | 17 + src/system/lite/src/dnodeMgmt.spec.c | 93 + src/system/lite/src/dnodeSystem.spec.c | 56 + src/system/lite/src/mgmtAcct.spec.c | 71 + src/system/lite/src/mgmtBalance.spec.c | 60 + src/system/lite/src/mgmtDnode.spec.c | 68 + .../src/mgmtDnodeInt.spec.c} | 157 +- src/system/lite/src/mgmtMnode.spec.c | 21 + src/system/lite/src/mgmtShell.spec.c | 48 + src/system/lite/src/mgmtSystem.spec.c | 45 + src/system/lite/src/vnodeFile.spec.c | 110 + src/system/lite/src/vnodePeer.spec.c | 41 + src/system/lite/src/vnodeStore.spec.c | 22 + src/system/src/mgmtDnode.c | 126 - src/system/src/mgmtDnodeInt.c | 340 -- src/system/src/vnodeFilterFunc.c | 470 -- src/util/CMakeLists.txt | 96 +- src/util/src/ihash.c | 3 +- src/util/src/lz4.c | 2361 ++++---- src/util/src/sql.c | 1914 ++++--- src/util/src/tcache.c | 12 +- src/util/src/tcrc32c.c | 19 +- src/util/src/textbuffer.c | 174 +- src/util/src/tglobalcfg.c | 505 +- src/util/src/tidpool.c | 35 + src/util/src/tlog.c | 4 +- src/util/src/tmd5.c | 5 +- src/util/src/tmem.c | 84 + src/util/src/tmempool.c | 6 +- src/util/src/tnote.c | 274 + src/util/src/tskiplist.c | 134 +- src/util/src/tstatus.c | 4 + src/util/src/tstoken.c | 78 +- src/util/src/tstrbuild.c | 83 + src/util/src/ttimer.c | 51 +- src/util/src/ttokenizer.c | 668 ++- src/util/src/ttypes.c | 318 +- src/util/src/tutil.c | 18 +- src/util/src/version.c | 6 +- tests/comparisonTest/cassandra/q1.txt | 22 +- tests/comparisonTest/cassandra/q2.txt | 100 +- tests/comparisonTest/cassandra/q3.txt | 20 +- tests/comparisonTest/cassandra/q4.txt | 20 +- tests/comparisonTest/influxdb/main.go | 558 +- tests/comparisonTest/influxdb/q1.txt | 22 +- tests/comparisonTest/influxdb/q2.txt | 122 +- tests/comparisonTest/influxdb/q3.txt | 22 +- tests/comparisonTest/influxdb/q4.txt | 22 +- .../opentsdb/opentsdbtest/pom.xml | 2 +- tests/comparisonTest/tdengine/q1.txt | 22 +- tests/comparisonTest/tdengine/q2.txt | 122 +- tests/comparisonTest/tdengine/q3.txt | 22 +- tests/comparisonTest/tdengine/q4.txt | 22 +- tests/examples/R/command.txt | 110 +- tests/examples/go/src/taosapp/taosapp.go | 0 tests/examples/lua/build.sh | 0 tests/examples/matlab/TDengineDemo.m | 256 +- tests/examples/python/read_example.py | 166 +- 247 files changed, 26608 insertions(+), 13963 deletions(-) mode change 100755 => 100644 packaging/deb/taosd mode change 100755 => 100644 packaging/rpm/taosd mode change 100755 => 100644 src/client/CMakeLists.txt create mode 100644 src/client/inc/tscJoinProcess.h mode change 100755 => 100644 src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h mode change 100755 => 100644 src/client/src/TSDBJNIConnector.c create mode 100644 src/client/src/tscJoinProcess.c create mode 100644 src/client/src/tscPrepare.c create mode 100644 src/connector/jdbc/src/test/java/TestPreparedStatement.java create mode 100644 src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java mode change 100755 => 100644 src/inc/buildInfo.h mode change 100755 => 100644 src/inc/sql.y create mode 100644 src/inc/tstrbuild.h mode change 100755 => 100644 src/modules/http/CMakeLists.txt mode change 100755 => 100644 src/modules/monitor/CMakeLists.txt mode change 100755 => 100644 src/rpc/CMakeLists.txt mode change 100755 => 100644 src/sdb/CMakeLists.txt mode change 100755 => 100644 src/system/CMakeLists.txt create mode 100644 src/system/detail/CMakeLists.txt rename src/system/{ => detail}/inc/dnodeSystem.h (65%) rename src/system/{ => detail}/inc/mgmt.h (80%) create mode 100644 src/system/detail/inc/mgmtBalance.h rename src/system/{ => detail}/inc/mgmtProfile.h (100%) rename src/system/{ => detail}/inc/mgmtSystem.h (70%) create mode 100644 src/system/detail/inc/mgmtUtil.h rename src/system/{ => detail}/inc/vnode.h (86%) rename src/system/{ => detail}/inc/vnodeCache.h (94%) rename src/system/{ => detail}/inc/vnodeDataFilterFunc.h (100%) rename src/system/{ => detail}/inc/vnodeFile.h (85%) rename src/system/{ => detail}/inc/vnodeMgmt.h (72%) create mode 100644 src/system/detail/inc/vnodePeer.h rename src/system/{ => detail}/inc/vnodeQueryImpl.h (79%) rename src/system/{ => detail}/inc/vnodeRead.h (76%) rename src/system/{ => detail}/inc/vnodeShell.h (100%) rename src/system/{ => detail}/inc/vnodeStore.h (88%) rename src/system/{ => detail}/inc/vnodeSystem.h (80%) rename src/system/{ => detail}/inc/vnodeTagMgmt.h (96%) rename src/system/{ => detail}/inc/vnodeUtil.h (94%) rename src/system/{ => detail}/src/dnodeMgmt.c (66%) rename src/system/{ => detail}/src/dnodeService.c (96%) rename src/system/{ => detail}/src/dnodeSystem.c (63%) rename src/system/{ => detail}/src/mgmtAcct.c (91%) rename src/system/{ => detail}/src/mgmtConn.c (99%) rename src/system/{ => detail}/src/mgmtDb.c (82%) create mode 100644 src/system/detail/src/mgmtDnode.c create mode 100644 src/system/detail/src/mgmtDnodeInt.c rename src/system/{ => detail}/src/mgmtMeter.c (77%) mode change 100755 => 100644 rename src/system/{ => detail}/src/mgmtProfile.c (99%) rename src/system/{ => detail}/src/mgmtShell.c (66%) create mode 100644 src/system/detail/src/mgmtSupertableQuery.c create mode 100644 src/system/detail/src/mgmtSystem.c rename src/system/{ => detail}/src/mgmtUser.c (93%) rename src/system/{ => detail}/src/mgmtUtil.c (51%) rename src/system/{ => detail}/src/mgmtVgroup.c (69%) rename src/system/{ => detail}/src/vnodeCache.c (83%) rename src/system/{ => detail}/src/vnodeCommit.c (98%) rename src/system/{ => detail}/src/vnodeFile.c (91%) create mode 100644 src/system/detail/src/vnodeFileUtil.c create mode 100644 src/system/detail/src/vnodeFilterFunc.c rename src/system/{ => detail}/src/vnodeImport.c (96%) rename src/system/{ => detail}/src/vnodeMeter.c (91%) rename src/system/{ => detail}/src/vnodeQueryImpl.c (75%) rename src/system/{ => detail}/src/vnodeQueryProcess.c (79%) rename src/system/{ => detail}/src/vnodeRead.c (83%) rename src/system/{ => detail}/src/vnodeShell.c (86%) rename src/system/{ => detail}/src/vnodeStore.c (85%) rename src/system/{ => detail}/src/vnodeStream.c (88%) rename src/system/{ => detail}/src/vnodeSystem.c (62%) rename src/system/{src/vnodeMeterTagMgmt.c => detail/src/vnodeTagMgmt.c} (81%) rename src/system/{ => detail}/src/vnodeUtil.c (73%) create mode 100644 src/system/lite/CMakeLists.txt create mode 100644 src/system/lite/src/dnodeMgmt.spec.c create mode 100644 src/system/lite/src/dnodeSystem.spec.c create mode 100644 src/system/lite/src/mgmtAcct.spec.c create mode 100644 src/system/lite/src/mgmtBalance.spec.c create mode 100644 src/system/lite/src/mgmtDnode.spec.c rename src/system/{src/mgmtSystem.c => lite/src/mgmtDnodeInt.spec.c} (52%) create mode 100644 src/system/lite/src/mgmtMnode.spec.c create mode 100644 src/system/lite/src/mgmtShell.spec.c create mode 100644 src/system/lite/src/mgmtSystem.spec.c create mode 100644 src/system/lite/src/vnodeFile.spec.c create mode 100644 src/system/lite/src/vnodePeer.spec.c create mode 100644 src/system/lite/src/vnodeStore.spec.c delete mode 100644 src/system/src/mgmtDnode.c delete mode 100644 src/system/src/mgmtDnodeInt.c delete mode 100644 src/system/src/vnodeFilterFunc.c mode change 100755 => 100644 src/util/CMakeLists.txt create mode 100644 src/util/src/tmem.c create mode 100644 src/util/src/tnote.c create mode 100644 src/util/src/tstrbuild.c mode change 100755 => 100644 src/util/src/version.c mode change 100755 => 100644 tests/examples/go/src/taosapp/taosapp.go mode change 100755 => 100644 tests/examples/lua/build.sh diff --git a/.gitignore b/.gitignore index 1cb4767e7c..dfc138d13f 100644 --- a/.gitignore +++ b/.gitignore @@ -23,5 +23,14 @@ tests/script/ tests/pytest/ tests/jenkins/ tests/hdfs/ - *.iml +*.class +nmake/ +sln/ +hdfs/ +c/ +taoshebei/ +taosdalipu/ +Target/ +*.failed +*.sql \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index af49d6811a..dcad242a30 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,171 +3,193 @@ PROJECT(TDengine) SET(CMAKE_C_STANDARD 11) SET(CMAKE_VERBOSE_MAKEFILE ON) -SET(TD_ROOT_DIR ${PROJECT_SOURCE_DIR}) # -# If it is a Linux operating system +# If need to set debug options # 1.Generate debug version: -# mkdir debug; cd debug; +# mkdir debug; cd debug; # cmake -DCMAKE_BUILD_TYPE=Debug .. # 2.Generate release version: -# mkdir release; cd release; -# cmake -DCMAKE_BUILD_TYPE=Release .. +# mkdir release; cd release; +# cmake -DCMAKE_BUILD_TYPE=Release .. +# + +# # If it is a Windows operating system # 1.Use command line tool of VS2013 or higher version # mkdir build; cd build; # cmake -G "NMake Makefiles" .. # nmake install -# 2.Use the VS development interface tool +# 2.Use the VS development interface tool # mkdir build; cd build; # cmake -A x64 .. # open the file named TDengine.sln # -# Set macro definitions according to os platform -SET(TD_WINDOWS FALSE) -SET(TD_LINUX FALSE) -SET(TD_ARM FALSE) -SET(TD_DARWIN FALSE) - -IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") - - SET(TD_OS_DIR ${PROJECT_SOURCE_DIR}/src/os/linux) - SET(TD_LINUX TRUE) - ADD_DEFINITIONS(-DLINUX) +IF (NOT DEFINED TD_CLUSTER) + MESSAGE(STATUS "Build the Lite Version") + SET(TD_CLUSTER FALSE) + SET(TD_LITE TRUE) + + SET(TD_COMMUNITY_DIR ${PROJECT_SOURCE_DIR}) + MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR}) + + + # Set macro definitions according to os platform + SET(TD_LINUX_64 FALSE) + SET(TD_LINUX_32 FALSE) + SET(TD_ARM_64 FALSE) + SET(TD_ARM_32 FALSE) + SET(TD_MIPS_64 FALSE) + SET(TD_DARWIN_64 FALSE) + SET(TD_WINDOWS_64 FALSE) + + IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) + SET(TD_LINUX_64 TRUE) + SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux) + ADD_DEFINITIONS(-D_M_X64) + MESSAGE(STATUS "The current platform is Linux 64-bit") + ELSE () + SET(TD_LINUX_32 TRUE) + MESSAGE(FATAL_ERROR "The current platform is Linux 32-bit, not supported yet") + EXIT () + ENDIF () + ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) + SET(TD_DARWIN_64 TRUE) + SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/darwin) + MESSAGE(STATUS "The current platform is Darwin 64-bit") + ELSE () + MESSAGE(FATAL_ERROR "The current platform is Darwin 32-bit, not supported yet") + EXIT () + ENDIF () + ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows") + IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) + SET(TD_WINDOWS_64 TRUE) + SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/windows) + ADD_DEFINITIONS(-D_M_X64) + MESSAGE(STATUS "The current platform is Windows 64-bit") + ELSE () + MESSAGE(FATAL_ERROR "The current platform is Windows 32-bit, not supported yet") + EXIT () + ENDIF () + ELSE() + MESSAGE(FATAL_ERROR "The current platform is not Linux/Darwin/Windows, stop compile") + EXIT () + ENDIF () - IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") - SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -Wno-char-subscripts -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + FIND_PROGRAM(TD_MVN_INSTALLED mvn) + IF (TD_MVN_INSTALLED) + MESSAGE(STATUS "MVN is installed and JDBC will be compiled") ELSE () - SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -Wno-char-subscripts -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + MESSAGE(STATUS "MVN is not installed and JDBC is not compiled") ENDIF () - SET(DEBUG_FLAGS "-O0 -DDEBUG") - SET(RELEASE_FLAGS "-O0") - - ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) - - IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) - MESSAGE(STATUS "The current platform is Linux 64-bit") - ADD_DEFINITIONS(-D_M_X64) + + # + # debug flag + # + # ADD_DEFINITIONS(-D_CHECK_HEADER_FILE_) + # ADD_DEFINITIONS(-D_TAOS_MEM_TEST_) + + IF (TD_CLUSTER) + ADD_DEFINITIONS(-DCLUSTER) + ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=3) ELSE () - MESSAGE(FATAL_ERROR "The current platform is Linux 32-bit, not supported yet") - EXIT () + ADD_DEFINITIONS(-DLITE) + ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1) ENDIF () - -ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows") - SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) - SET(TD_OS_DIR ${PROJECT_SOURCE_DIR}/src/os/windows) - SET(TD_WINDOWS TRUE) - ADD_DEFINITIONS(-DWINDOWS) - ADD_DEFINITIONS(-D__CLEANUP_C) - ADD_DEFINITIONS(-DPTW32_STATIC_LIB) - ADD_DEFINITIONS(-DPTW32_BUILD) - - SET(COMMON_FLAGS "/nologo /WX- /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") - SET(DEBUG_FLAGS "/Zi /W3 /GL") - SET(RELEASE_FLAGS "/W0 /GL") - - ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) - - IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) - MESSAGE(STATUS "The current platform is Windows 64-bit") - ADD_DEFINITIONS(-D_M_X64) + IF (TD_LINUX_64) + SET(DEBUG_FLAGS "-O0 -DDEBUG") + SET(RELEASE_FLAGS "-O0") + IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") + SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -Wno-char-subscripts -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + ELSE () + SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -Wno-char-subscripts -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + ENDIF () + ADD_DEFINITIONS(-DLINUX) + ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) + ELSEIF (TD_WINDOWS_64) + SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) + SET(COMMON_FLAGS "/nologo /WX- /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") + SET(DEBUG_FLAGS "/Zi /W3 /GL") + SET(RELEASE_FLAGS "/W0 /GL") + ADD_DEFINITIONS(-DWINDOWS) + ADD_DEFINITIONS(-D__CLEANUP_C) + ADD_DEFINITIONS(-DPTW32_STATIC_LIB) + ADD_DEFINITIONS(-DPTW32_BUILD) + ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) + ELSEIF (TD_DARWIN_64) + SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -Wno-char-subscripts -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-unused-variable -Wno-bitfield-constant-conversion") + SET(DEBUG_FLAGS "-O0 -DDEBUG") + SET(RELEASE_FLAGS "-O0") + ADD_DEFINITIONS(-DDARWIN) + ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) ELSE () - MESSAGE(FATAL_ERROR "The current platform is Windows 32-bit, not supported yet") + MESSAGE(FATAL_ERROR "The current platform is not support yet, stop compile") EXIT () ENDIF () -ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + # Set compiler options + SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${COMMON_FLAGS} ${DEBUG_FLAGS}") + SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_FLAGS} ${RELEASE_FLAGS}") - SET(TD_OS_DIR ${PROJECT_SOURCE_DIR}/src/os/darwin) - SET(TD_DARWIN TRUE) - ADD_DEFINITIONS(-DDARWIN) + # Set c++ compiler options + # SET(COMMON_CXX_FLAGS "${COMMON_FLAGS} -std=c++11") + # SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMMON_CXX_FLAGS} ${DEBUG_FLAGS}") + # SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMMON_CXX_FLAGS} ${RELEASE_FLAGS}") - SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -Wno-char-subscripts -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-unused-variable -Wno-bitfield-constant-conversion") - SET(DEBUG_FLAGS "-O0 -DDEBUG") - SET(RELEASE_FLAGS "-O0") - - ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) - - IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) - MESSAGE(STATUS "The current platform is Darwin 64-bit") - ADD_DEFINITIONS(-D_M_X64) + IF (${CMAKE_BUILD_TYPE} MATCHES "Debug") + MESSAGE(STATUS "Build Debug Version") + ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release") + MESSAGE(STATUS "Build Release Version") ELSE () - MESSAGE(FATAL_ERROR "The current platform is Darwin 32-bit, not supported yet") - EXIT () + IF (TD_WINDOWS_64) + SET(CMAKE_BUILD_TYPE "Release") + MESSAGE(STATUS "Build Release Version in Windows as default") + ELSE () + SET(CMAKE_BUILD_TYPE "Debug") + MESSAGE(STATUS "Build Debug Version as default") + ENDIF() ENDIF () -ELSE () - MESSAGE(FATAL_ERROR "The current platform is not Linux/MAC/Windows, stop compile") - EXIT () - -ENDIF () - -# Set compiler options -SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${COMMON_FLAGS} ${DEBUG_FLAGS}") -SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_FLAGS} ${RELEASE_FLAGS}") - -# Set c++ compiler options -# SET(COMMON_CXX_FLAGS "${COMMON_FLAGS} -std=c++11") -# SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMMON_CXX_FLAGS} ${DEBUG_FLAGS}") -# SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMMON_CXX_FLAGS} ${RELEASE_FLAGS}") - -IF (${CMAKE_BUILD_TYPE} MATCHES "Debug") - MESSAGE(STATUS "Build Debug Version") -ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release") - MESSAGE(STATUS "Build Release Version") -ELSE () - IF (TD_WINDOWS) - SET(CMAKE_BUILD_TYPE "Release") - MESSAGE(STATUS "Build Release Version") - ELSE () - SET(CMAKE_BUILD_TYPE "Debug") - MESSAGE(STATUS "Build Debug Version") - ENDIF() -ENDIF () - -FIND_PROGRAM(TD_MVN_INSTALLED mvn) -IF (TD_MVN_INSTALLED) - MESSAGE(STATUS "MVN is installed and JDBC will be compiled") -ELSE () - MESSAGE(STATUS "MVN is not installed and JDBC is not compiled") + #set output directory + SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) + SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin) + SET(TD_TESTS_OUTPUT_DIR ${PROJECT_BINARY_DIR}/test) + + MESSAGE(STATUS "Operating system dependency directory: " ${TD_OS_DIR}) + MESSAGE(STATUS "Project source directory: " ${PROJECT_SOURCE_DIR}) + MESSAGE(STATUS "Project binary files output path: " ${PROJECT_BINARY_DIR}) + MESSAGE(STATUS "Project executable files output path: " ${EXECUTABLE_OUTPUT_PATH}) + MESSAGE(STATUS "Project library files output path: " ${LIBRARY_OUTPUT_PATH}) + + IF (TD_LINUX_64) + SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") + INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") + INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})") + INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR})") + ELSEIF (TD_WINDOWS_64) + SET(CMAKE_INSTALL_PREFIX C:/TDengine) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/grafana DESTINATION connector) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .) + INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .) + #INSTALL(TARGETS taos RUNTIME DESTINATION driver) + #INSTALL(TARGETS shell RUNTIME DESTINATION .) + IF (TD_MVN_INSTALLED) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-1.0.2-dist.jar DESTINATION connector/jdbc) + ENDIF () + ENDIF () ENDIF () -#set output directory -SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) -SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin) -SET(TD_TESTS_OUTPUT_DIR ${PROJECT_BINARY_DIR}/test) - -MESSAGE(STATUS "Operating system dependency directory: " ${TD_OS_DIR}) -MESSAGE(STATUS "Project source directory: " ${PROJECT_SOURCE_DIR}) -MESSAGE(STATUS "Project binary files output path: " ${PROJECT_BINARY_DIR}) -MESSAGE(STATUS "Project executable files output path: " ${EXECUTABLE_OUTPUT_PATH}) -MESSAGE(STATUS "Project library files output path: " ${LIBRARY_OUTPUT_PATH}) - ADD_SUBDIRECTORY(deps) ADD_SUBDIRECTORY(src) -ADD_SUBDIRECTORY(tests) - -IF (TD_LINUX) - SET(TD_MAKE_INSTALL_SH "${TD_ROOT_DIR}/packaging/tools/make_install.sh") - INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") - INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})") - INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_ROOT_DIR} ${PROJECT_BINARY_DIR})") -ELSEIF (TD_WINDOWS) - SET(CMAKE_INSTALL_PREFIX C:/TDengine) - INSTALL(DIRECTORY ${TD_ROOT_DIR}/src/connector/go DESTINATION connector) - INSTALL(DIRECTORY ${TD_ROOT_DIR}/src/connector/grafana DESTINATION connector) - INSTALL(DIRECTORY ${TD_ROOT_DIR}/src/connector/python DESTINATION connector) - INSTALL(DIRECTORY ${TD_ROOT_DIR}/tests/examples DESTINATION .) - INSTALL(DIRECTORY ${TD_ROOT_DIR}/packaging/cfg DESTINATION .) - INSTALL(FILES ${TD_ROOT_DIR}/src/inc/taos.h DESTINATION include) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver) - INSTALL(TARGETS taos RUNTIME DESTINATION driver) - INSTALL(TARGETS shell RUNTIME DESTINATION .) - IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-1.0.2-dist.jar DESTINATION connector/jdbc) - ENDIF () -ENDIF () -INCLUDE(CPack) \ No newline at end of file +INCLUDE(CPack) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index df76843a3d..bc5bd8c037 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -1,5 +1,4 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) ADD_SUBDIRECTORY(zlib-1.2.11) diff --git a/deps/iconv/CMakeLists.txt b/deps/iconv/CMakeLists.txt index 8af4beafc0..944cb400e0 100644 --- a/deps/iconv/CMakeLists.txt +++ b/deps/iconv/CMakeLists.txt @@ -1,8 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -IF (TD_WINDOWS) +IF (TD_WINDOWS_64) LIST(APPEND SRC iconv.c) LIST(APPEND SRC localcharset.c) INCLUDE_DIRECTORIES(.) diff --git a/deps/pthread/CMakeLists.txt b/deps/pthread/CMakeLists.txt index 1499fa099c..a218d05bab 100644 --- a/deps/pthread/CMakeLists.txt +++ b/deps/pthread/CMakeLists.txt @@ -1,9 +1,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -IF (TD_WINDOWS) - LIST(APPEND SRC pthread.c) +IF (TD_WINDOWS_64) INCLUDE_DIRECTORIES(.) + LIST(APPEND SRC pthread.c) ADD_LIBRARY(pthread ${SRC}) ENDIF () diff --git a/deps/regex/CMakeLists.txt b/deps/regex/CMakeLists.txt index a7c373fd07..8f8396caa4 100644 --- a/deps/regex/CMakeLists.txt +++ b/deps/regex/CMakeLists.txt @@ -1,9 +1,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -IF (TD_WINDOWS) - LIST(APPEND SRC regex.c) +IF (TD_WINDOWS_64) INCLUDE_DIRECTORIES(inc .) + LIST(APPEND SRC regex.c) ADD_LIBRARY(regex ${SRC}) ENDIF () diff --git a/deps/zlib-1.2.11/CMakeLists.txt b/deps/zlib-1.2.11/CMakeLists.txt index cd4c0173fb..c69e54a1b0 100644 --- a/deps/zlib-1.2.11/CMakeLists.txt +++ b/deps/zlib-1.2.11/CMakeLists.txt @@ -1,9 +1,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -IF (TD_LINUX) - AUX_SOURCE_DIRECTORY(src SRC) +IF (TD_LINUX_64) INCLUDE_DIRECTORIES(inc) + AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(z ${SRC}) ENDIF () diff --git a/deps/zlib-1.2.11/inc/zconf.h b/deps/zlib-1.2.11/inc/zconf.h index 77398c11a1..97b1e279ef 100644 --- a/deps/zlib-1.2.11/inc/zconf.h +++ b/deps/zlib-1.2.11/inc/zconf.h @@ -4,7 +4,7 @@ */ /* @(#) $Id$ */ - +//#include #ifndef ZCONF_H #define ZCONF_H @@ -238,16 +238,16 @@ #endif #ifdef Z_SOLO - typedef unsigned long z_size_t; + typedef uint64_t z_size_t; #else -# define z_longlong long long +# define z_longlong int64_t # if defined(NO_SIZE_T) typedef unsigned NO_SIZE_T z_size_t; # elif defined(STDC) # include typedef size_t z_size_t; # else - typedef unsigned long z_size_t; + typedef uint64_t z_size_t; # endif # undef z_longlong #endif @@ -391,7 +391,7 @@ typedef unsigned char Byte; /* 8 bits */ #endif typedef unsigned int uInt; /* 16 bits or more */ -typedef unsigned long uLong; /* 32 bits or more */ +typedef uint64_t uLong; /* 32 bits or more */ #ifdef SMALL_MEDIUM /* Borland C/C++ and some old MSC versions ignore FAR inside typedef */ @@ -419,7 +419,7 @@ typedef uLong FAR uLongf; # if (UINT_MAX == 0xffffffffUL) # define Z_U4 unsigned # elif (ULONG_MAX == 0xffffffffUL) -# define Z_U4 unsigned long +# define Z_U4 uint64_t # elif (USHRT_MAX == 0xffffffffUL) # define Z_U4 unsigned short # endif @@ -428,7 +428,7 @@ typedef uLong FAR uLongf; #ifdef Z_U4 typedef Z_U4 z_crc_t; #else - typedef unsigned long z_crc_t; + typedef uint64_t z_crc_t; #endif #if 1 /* was set to #if 1 by ./configure */ @@ -501,7 +501,7 @@ typedef uLong FAR uLongf; #endif #ifndef z_off_t -# define z_off_t long +# define z_off_t int64_t #endif #if !defined(_WIN32) && defined(Z_LARGE64) diff --git a/deps/zlib-1.2.11/inc/zlib.h b/deps/zlib-1.2.11/inc/zlib.h index f09cdaf1e0..f407de4233 100644 --- a/deps/zlib-1.2.11/inc/zlib.h +++ b/deps/zlib-1.2.11/inc/zlib.h @@ -999,7 +999,7 @@ ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm, stream state was inconsistent. */ -ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm)); +ZEXTERN int64_t ZEXPORT inflateMark OF((z_streamp strm)); /* This function returns two values, one in the lower 16 bits of the return value, and the other in the remaining upper bits, obtained by shifting the @@ -1890,7 +1890,7 @@ ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp)); ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table OF((void)); ZEXTERN int ZEXPORT inflateUndermine OF((z_streamp, int)); ZEXTERN int ZEXPORT inflateValidate OF((z_streamp, int)); -ZEXTERN unsigned long ZEXPORT inflateCodesUsed OF ((z_streamp)); +ZEXTERN uint64_t ZEXPORT inflateCodesUsed OF ((z_streamp)); ZEXTERN int ZEXPORT inflateResetKeep OF((z_streamp)); ZEXTERN int ZEXPORT deflateResetKeep OF((z_streamp)); #if (defined(_WIN32) || defined(__CYGWIN__)) && !defined(Z_SOLO) diff --git a/deps/zlib-1.2.11/src/adler32.c b/deps/zlib-1.2.11/src/adler32.c index d0be4380a3..73b2a8431f 100644 --- a/deps/zlib-1.2.11/src/adler32.c +++ b/deps/zlib-1.2.11/src/adler32.c @@ -4,7 +4,7 @@ */ /* @(#) $Id$ */ - +#include #include "zutil.h" local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); @@ -26,7 +26,7 @@ local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); (thank you to John Reiser for pointing this out) */ # define CHOP(a) \ do { \ - unsigned long tmp = a >> 16; \ + uint64_t tmp = a >> 16; \ a &= 0xffffUL; \ a += (tmp << 4) - tmp; \ } while (0) @@ -65,7 +65,7 @@ uLong ZEXPORT adler32_z(adler, buf, len) const Bytef *buf; z_size_t len; { - unsigned long sum2; + uint64_t sum2; unsigned n; /* split Adler-32 into component sums */ @@ -145,8 +145,8 @@ local uLong adler32_combine_(adler1, adler2, len2) uLong adler2; z_off64_t len2; { - unsigned long sum1; - unsigned long sum2; + uint64_t sum1; + uint64_t sum2; unsigned rem; /* for negative len, return invalid adler32 as a clue for debugging */ @@ -163,7 +163,7 @@ local uLong adler32_combine_(adler1, adler2, len2) sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem; if (sum1 >= BASE) sum1 -= BASE; if (sum1 >= BASE) sum1 -= BASE; - if (sum2 >= ((unsigned long)BASE << 1)) sum2 -= ((unsigned long)BASE << 1); + if (sum2 >= ((uint64_t)BASE << 1)) sum2 -= ((uint64_t)BASE << 1); if (sum2 >= BASE) sum2 -= BASE; return sum1 | (sum2 << 16); } diff --git a/deps/zlib-1.2.11/src/compress.c b/deps/zlib-1.2.11/src/compress.c index e2db404abf..251519572a 100644 --- a/deps/zlib-1.2.11/src/compress.c +++ b/deps/zlib-1.2.11/src/compress.c @@ -4,7 +4,7 @@ */ /* @(#) $Id$ */ - +#include #define ZLIB_INTERNAL #include "zlib.h" diff --git a/deps/zlib-1.2.11/src/crc32.c b/deps/zlib-1.2.11/src/crc32.c index 9580440c0e..400d1a1011 100644 --- a/deps/zlib-1.2.11/src/crc32.c +++ b/deps/zlib-1.2.11/src/crc32.c @@ -20,7 +20,7 @@ DYNAMIC_CRC_TABLE and MAKECRCH can be #defined to write out crc32.h. */ - +#include #ifdef MAKECRCH # include # ifndef DYNAMIC_CRC_TABLE @@ -35,9 +35,9 @@ # define BYFOUR #endif #ifdef BYFOUR - local unsigned long crc32_little OF((unsigned long, + local uint64_t crc32_little OF((uint64_t, const unsigned char FAR *, z_size_t)); - local unsigned long crc32_big OF((unsigned long, + local uint64_t crc32_big OF((uint64_t, const unsigned char FAR *, z_size_t)); # define TBLS 8 #else @@ -45,9 +45,9 @@ #endif /* BYFOUR */ /* Local functions for crc concatenation */ -local unsigned long gf2_matrix_times OF((unsigned long *mat, - unsigned long vec)); -local void gf2_matrix_square OF((unsigned long *square, unsigned long *mat)); +local uint64_t gf2_matrix_times OF((uint64_t *mat, + uint64_t vec)); +local void gf2_matrix_square OF((uint64_t *square, uint64_t *mat)); local uLong crc32_combine_ OF((uLong crc1, uLong crc2, z_off64_t len2)); @@ -170,7 +170,7 @@ local void write_table(out, table) for (n = 0; n < 256; n++) fprintf(out, "%s0x%08lxUL%s", n % 5 ? "" : " ", - (unsigned long)(table[n]), + (uint64_t)(table[n]), n == 255 ? "\n" : (n % 5 == 4 ? ",\n" : ", ")); } #endif /* MAKECRCH */ @@ -199,8 +199,8 @@ const z_crc_t FAR * ZEXPORT get_crc_table() #define DO8 DO1; DO1; DO1; DO1; DO1; DO1; DO1; DO1 /* ========================================================================= */ -unsigned long ZEXPORT crc32_z(crc, buf, len) - unsigned long crc; +uint64_t ZEXPORT crc32_z(crc, buf, len) + uint64_t crc; const unsigned char FAR *buf; z_size_t len; { @@ -234,8 +234,8 @@ unsigned long ZEXPORT crc32_z(crc, buf, len) } /* ========================================================================= */ -unsigned long ZEXPORT crc32(crc, buf, len) - unsigned long crc; +uint64_t ZEXPORT crc32(crc, buf, len) + uint64_t crc; const unsigned char FAR *buf; uInt len; { @@ -263,8 +263,8 @@ unsigned long ZEXPORT crc32(crc, buf, len) #define DOLIT32 DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4 /* ========================================================================= */ -local unsigned long crc32_little(crc, buf, len) - unsigned long crc; +local uint64_t crc32_little(crc, buf, len) + uint64_t crc; const unsigned char FAR *buf; z_size_t len; { @@ -293,7 +293,7 @@ local unsigned long crc32_little(crc, buf, len) c = crc_table[0][(c ^ *buf++) & 0xff] ^ (c >> 8); } while (--len); c = ~c; - return (unsigned long)c; + return (uint64_t)c; } /* ========================================================================= */ @@ -303,8 +303,8 @@ local unsigned long crc32_little(crc, buf, len) #define DOBIG32 DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4 /* ========================================================================= */ -local unsigned long crc32_big(crc, buf, len) - unsigned long crc; +local uint64_t crc32_big(crc, buf, len) + uint64_t crc; const unsigned char FAR *buf; z_size_t len; { @@ -333,7 +333,7 @@ local unsigned long crc32_big(crc, buf, len) c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8); } while (--len); c = ~c; - return (unsigned long)(ZSWAP32(c)); + return (uint64_t)(ZSWAP32(c)); } #endif /* BYFOUR */ @@ -341,11 +341,11 @@ local unsigned long crc32_big(crc, buf, len) #define GF2_DIM 32 /* dimension of GF(2) vectors (length of CRC) */ /* ========================================================================= */ -local unsigned long gf2_matrix_times(mat, vec) - unsigned long *mat; - unsigned long vec; +local uint64_t gf2_matrix_times(mat, vec) + uint64_t *mat; + uint64_t vec; { - unsigned long sum; + uint64_t sum; sum = 0; while (vec) { @@ -359,8 +359,8 @@ local unsigned long gf2_matrix_times(mat, vec) /* ========================================================================= */ local void gf2_matrix_square(square, mat) - unsigned long *square; - unsigned long *mat; + uint64_t *square; + uint64_t *mat; { int n; @@ -375,9 +375,9 @@ local uLong crc32_combine_(crc1, crc2, len2) z_off64_t len2; { int n; - unsigned long row; - unsigned long even[GF2_DIM]; /* even-power-of-two zeros operator */ - unsigned long odd[GF2_DIM]; /* odd-power-of-two zeros operator */ + uint64_t row; + uint64_t even[GF2_DIM]; /* even-power-of-two zeros operator */ + uint64_t odd[GF2_DIM]; /* odd-power-of-two zeros operator */ /* degenerate case (also disallow negative lengths) */ if (len2 <= 0) diff --git a/deps/zlib-1.2.11/src/deflate.c b/deps/zlib-1.2.11/src/deflate.c index 1ec761448d..2a3773a929 100644 --- a/deps/zlib-1.2.11/src/deflate.c +++ b/deps/zlib-1.2.11/src/deflate.c @@ -48,7 +48,7 @@ */ /* @(#) $Id$ */ - +#include #include "deflate.h" const char deflate_copyright[] = @@ -430,7 +430,7 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) fill_window(s); } s->strstart += s->lookahead; - s->block_start = (long)s->strstart; + s->block_start = (int64_t)s->strstart; s->insert = s->lookahead; s->lookahead = 0; s->match_length = s->prev_length = MIN_MATCH-1; @@ -1512,7 +1512,7 @@ local void fill_window(s) zmemcpy(s->window, s->window+wsize, (unsigned)wsize - more); s->match_start -= wsize; s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ - s->block_start -= (long) wsize; + s->block_start -= (int64_t) wsize; slide_hash(s); more += wsize; } @@ -1606,7 +1606,7 @@ local void fill_window(s) _tr_flush_block(s, (s->block_start >= 0L ? \ (charf *)&s->window[(unsigned)s->block_start] : \ (charf *)Z_NULL), \ - (ulg)((long)s->strstart - s->block_start), \ + (ulg)((int64_t)s->strstart - s->block_start), \ (last)); \ s->block_start = s->strstart; \ flush_pending(s->strm); \ @@ -1766,12 +1766,12 @@ local block_state deflate_stored(s, flush) /* If flushing and all input has been consumed, then done. */ if (flush != Z_NO_FLUSH && flush != Z_FINISH && - s->strm->avail_in == 0 && (long)s->strstart == s->block_start) + s->strm->avail_in == 0 && (int64_t)s->strstart == s->block_start) return block_done; /* Fill the window with any remaining input. */ have = s->window_size - s->strstart - 1; - if (s->strm->avail_in > have && s->block_start >= (long)s->w_size) { + if (s->strm->avail_in > have && s->block_start >= (int64_t)s->w_size) { /* Slide the window down. */ s->block_start -= s->w_size; s->strstart -= s->w_size; diff --git a/deps/zlib-1.2.11/src/deflate.h b/deps/zlib-1.2.11/src/deflate.h index 23ecdd312b..e090fc7a86 100644 --- a/deps/zlib-1.2.11/src/deflate.h +++ b/deps/zlib-1.2.11/src/deflate.h @@ -151,7 +151,7 @@ typedef struct internal_state { * hash_shift * MIN_MATCH >= hash_bits */ - long block_start; + int64_t block_start; /* Window position at the beginning of the current output block. Gets * negative when the window is moved backwards. */ diff --git a/deps/zlib-1.2.11/src/gzclose.c b/deps/zlib-1.2.11/src/gzclose.c index caeb99a317..154fb3a0d4 100644 --- a/deps/zlib-1.2.11/src/gzclose.c +++ b/deps/zlib-1.2.11/src/gzclose.c @@ -2,7 +2,7 @@ * Copyright (C) 2004, 2010 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ - +#include #include "gzguts.h" /* gzclose() is in a separate file so that it is linked in only if it is used. diff --git a/deps/zlib-1.2.11/src/gzlib.c b/deps/zlib-1.2.11/src/gzlib.c index 4105e6aff9..381cdc2c7d 100644 --- a/deps/zlib-1.2.11/src/gzlib.c +++ b/deps/zlib-1.2.11/src/gzlib.c @@ -2,7 +2,7 @@ * Copyright (C) 2004-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ - +#include #include "gzguts.h" #if defined(_WIN32) && !defined(__BORLANDC__) && !defined(__MINGW32__) diff --git a/deps/zlib-1.2.11/src/gzread.c b/deps/zlib-1.2.11/src/gzread.c index 956b91ea7d..bb6a17600c 100644 --- a/deps/zlib-1.2.11/src/gzread.c +++ b/deps/zlib-1.2.11/src/gzread.c @@ -2,7 +2,7 @@ * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ - +#include #include "gzguts.h" /* Local functions */ diff --git a/deps/zlib-1.2.11/src/gzwrite.c b/deps/zlib-1.2.11/src/gzwrite.c index c7b5651d70..e19faecd5d 100644 --- a/deps/zlib-1.2.11/src/gzwrite.c +++ b/deps/zlib-1.2.11/src/gzwrite.c @@ -2,7 +2,7 @@ * Copyright (C) 2004-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ - +#include #include "gzguts.h" /* Local functions */ diff --git a/deps/zlib-1.2.11/src/infback.c b/deps/zlib-1.2.11/src/infback.c index 59679ecbfc..f63274ef75 100644 --- a/deps/zlib-1.2.11/src/infback.c +++ b/deps/zlib-1.2.11/src/infback.c @@ -9,7 +9,7 @@ with inffast.c is retained so that optimized assembler-coded versions of inflate_fast() can be used with either inflate.c or infback.c. */ - +#include #include "zutil.h" #include "inftrees.h" #include "inflate.h" @@ -173,7 +173,7 @@ struct inflate_state FAR *state; do { \ PULL(); \ have--; \ - hold += (unsigned long)(*next++) << bits; \ + hold += (uint64_t)(*next++) << bits; \ bits += 8; \ } while (0) @@ -258,7 +258,7 @@ void FAR *out_desc; z_const unsigned char FAR *next; /* next input */ unsigned char FAR *put; /* next output */ unsigned have, left; /* available input and output */ - unsigned long hold; /* bit buffer */ + uint64_t hold; /* bit buffer */ unsigned bits; /* bits in bit buffer */ unsigned copy; /* number of stored or match bytes to copy */ unsigned char FAR *from; /* where to copy match bytes from */ diff --git a/deps/zlib-1.2.11/src/inffast.c b/deps/zlib-1.2.11/src/inffast.c index 0dbd1dbc09..bec6441f5b 100644 --- a/deps/zlib-1.2.11/src/inffast.c +++ b/deps/zlib-1.2.11/src/inffast.c @@ -2,7 +2,7 @@ * Copyright (C) 1995-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ - +#include #include "zutil.h" #include "inftrees.h" #include "inflate.h" @@ -64,7 +64,7 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ unsigned whave; /* valid bytes in the window */ unsigned wnext; /* window write index */ unsigned char FAR *window; /* allocated sliding window, if wsize != 0 */ - unsigned long hold; /* local strm->hold */ + uint64_t hold; /* local strm->hold */ unsigned bits; /* local strm->bits */ code const FAR *lcode; /* local strm->lencode */ code const FAR *dcode; /* local strm->distcode */ @@ -102,9 +102,9 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ input data or output space */ do { if (bits < 15) { - hold += (unsigned long)(*in++) << bits; + hold += (uint64_t)(*in++) << bits; bits += 8; - hold += (unsigned long)(*in++) << bits; + hold += (uint64_t)(*in++) << bits; bits += 8; } here = lcode[hold & lmask]; @@ -124,7 +124,7 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ op &= 15; /* number of extra bits */ if (op) { if (bits < op) { - hold += (unsigned long)(*in++) << bits; + hold += (uint64_t)(*in++) << bits; bits += 8; } len += (unsigned)hold & ((1U << op) - 1); @@ -133,9 +133,9 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ } Tracevv((stderr, "inflate: length %u\n", len)); if (bits < 15) { - hold += (unsigned long)(*in++) << bits; + hold += (uint64_t)(*in++) << bits; bits += 8; - hold += (unsigned long)(*in++) << bits; + hold += (uint64_t)(*in++) << bits; bits += 8; } here = dcode[hold & dmask]; @@ -148,10 +148,10 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ dist = (unsigned)(here.val); op &= 15; /* number of extra bits */ if (bits < op) { - hold += (unsigned long)(*in++) << bits; + hold += (uint64_t)(*in++) << bits; bits += 8; if (bits < op) { - hold += (unsigned long)(*in++) << bits; + hold += (uint64_t)(*in++) << bits; bits += 8; } } diff --git a/deps/zlib-1.2.11/src/inflate.c b/deps/zlib-1.2.11/src/inflate.c index ac333e8c2e..d078430ef1 100644 --- a/deps/zlib-1.2.11/src/inflate.c +++ b/deps/zlib-1.2.11/src/inflate.c @@ -79,7 +79,7 @@ * * The history for versions after 1.2.0 are in ChangeLog in zlib distribution. */ - +#include #include "zutil.h" #include "inftrees.h" #include "inflate.h" @@ -507,7 +507,7 @@ unsigned copy; do { \ if (have == 0) goto inf_leave; \ have--; \ - hold += (unsigned long)(*next++) << bits; \ + hold += (uint64_t)(*next++) << bits; \ bits += 8; \ } while (0) @@ -627,7 +627,7 @@ int flush; z_const unsigned char FAR *next; /* next input */ unsigned char FAR *put; /* next output */ unsigned have, left; /* available input and output */ - unsigned long hold; /* bit buffer */ + uint64_t hold; /* bit buffer */ unsigned bits; /* bits in bit buffer */ unsigned in, out; /* save starting available input and output */ unsigned copy; /* number of stored or match bytes to copy */ @@ -1317,7 +1317,7 @@ const Bytef *dictionary; uInt dictLength; { struct inflate_state FAR *state; - unsigned long dictid; + uint64_t dictid; int ret; /* check state */ @@ -1401,7 +1401,7 @@ int ZEXPORT inflateSync(strm) z_streamp strm; { unsigned len; /* number of bytes to look at or looked at */ - unsigned long in, out; /* temporary to save total_in and total_out */ + uint64_t in, out; /* temporary to save total_in and total_out */ unsigned char buf[4]; /* to restore bit buffer to byte string */ struct inflate_state FAR *state; @@ -1538,7 +1538,7 @@ int check; return Z_OK; } -long ZEXPORT inflateMark(strm) +int64_t ZEXPORT inflateMark(strm) z_streamp strm; { struct inflate_state FAR *state; @@ -1546,16 +1546,16 @@ z_streamp strm; if (inflateStateCheck(strm)) return -(1L << 16); state = (struct inflate_state FAR *)strm->state; - return (long)(((unsigned long)((long)state->back)) << 16) + + return (int64_t)(((uint64_t)((int64_t)state->back)) << 16) + (state->mode == COPY ? state->length : (state->mode == MATCH ? state->was - state->length : 0)); } -unsigned long ZEXPORT inflateCodesUsed(strm) +uint64_t ZEXPORT inflateCodesUsed(strm) z_streamp strm; { struct inflate_state FAR *state; - if (inflateStateCheck(strm)) return (unsigned long)-1; + if (inflateStateCheck(strm)) return (uint64_t)-1; state = (struct inflate_state FAR *)strm->state; - return (unsigned long)(state->next - state->codes); + return (uint64_t)(state->next - state->codes); } diff --git a/deps/zlib-1.2.11/src/inflate.h b/deps/zlib-1.2.11/src/inflate.h index a46cce6b6d..7d05e72ff4 100644 --- a/deps/zlib-1.2.11/src/inflate.h +++ b/deps/zlib-1.2.11/src/inflate.h @@ -88,8 +88,8 @@ struct inflate_state { int havedict; /* true if dictionary provided */ int flags; /* gzip header method and flags (0 if zlib) */ unsigned dmax; /* zlib header max distance (INFLATE_STRICT) */ - unsigned long check; /* protected copy of check value */ - unsigned long total; /* protected copy of output count */ + uint64_t check; /* protected copy of check value */ + uint64_t total; /* protected copy of output count */ gz_headerp head; /* where to save gzip header information */ /* sliding window */ unsigned wbits; /* log base 2 of requested window size */ @@ -98,7 +98,7 @@ struct inflate_state { unsigned wnext; /* window write index */ unsigned char FAR *window; /* allocated sliding window, if needed */ /* bit accumulator */ - unsigned long hold; /* input bit accumulator */ + uint64_t hold; /* input bit accumulator */ unsigned bits; /* number of bits in "in" */ /* for string and stored block copying */ unsigned length; /* literal or length of data to copy */ diff --git a/deps/zlib-1.2.11/src/inftrees.c b/deps/zlib-1.2.11/src/inftrees.c index 2ea08fc13e..f2cb5662fe 100644 --- a/deps/zlib-1.2.11/src/inftrees.c +++ b/deps/zlib-1.2.11/src/inftrees.c @@ -2,7 +2,7 @@ * Copyright (C) 1995-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ - +#include #include "zutil.h" #include "inftrees.h" diff --git a/deps/zlib-1.2.11/src/trees.c b/deps/zlib-1.2.11/src/trees.c index 50cf4b4571..f558ae226c 100644 --- a/deps/zlib-1.2.11/src/trees.c +++ b/deps/zlib-1.2.11/src/trees.c @@ -33,7 +33,7 @@ /* @(#) $Id$ */ /* #define GEN_TREES_H */ - +#include #include "deflate.h" #ifdef ZLIB_DEBUG @@ -1038,7 +1038,7 @@ int ZLIB_INTERNAL _tr_tally (s, dist, lc) if ((s->last_lit & 0x1fff) == 0 && s->level > 2) { /* Compute an upper bound for the compressed length */ ulg out_length = (ulg)s->last_lit*8L; - ulg in_length = (ulg)((long)s->strstart - s->block_start); + ulg in_length = (ulg)((int64_t)s->strstart - s->block_start); int dcode; for (dcode = 0; dcode < D_CODES; dcode++) { out_length += (ulg)s->dyn_dtree[dcode].Freq * @@ -1128,7 +1128,7 @@ local int detect_data_type(s) * set bits 0..6, 14..25, and 28..31 * 0xf3ffc07f = binary 11110011111111111100000001111111 */ - unsigned long black_mask = 0xf3ffc07fUL; + uint64_t black_mask = 0xf3ffc07fUL; int n; /* Check for non-textual ("black-listed") bytes. */ diff --git a/deps/zlib-1.2.11/src/uncompr.c b/deps/zlib-1.2.11/src/uncompr.c index f03a1a865e..b46a44310e 100644 --- a/deps/zlib-1.2.11/src/uncompr.c +++ b/deps/zlib-1.2.11/src/uncompr.c @@ -4,7 +4,7 @@ */ /* @(#) $Id$ */ - +#include #define ZLIB_INTERNAL #include "zlib.h" diff --git a/deps/zlib-1.2.11/src/zconf.h b/deps/zlib-1.2.11/src/zconf.h index 77398c11a1..d71de993e1 100644 --- a/deps/zlib-1.2.11/src/zconf.h +++ b/deps/zlib-1.2.11/src/zconf.h @@ -238,16 +238,16 @@ #endif #ifdef Z_SOLO - typedef unsigned long z_size_t; + typedef uint64_t z_size_t; #else -# define z_longlong long long +# define z_longlong int64_t # if defined(NO_SIZE_T) typedef unsigned NO_SIZE_T z_size_t; # elif defined(STDC) # include typedef size_t z_size_t; # else - typedef unsigned long z_size_t; + typedef uint64_t z_size_t; # endif # undef z_longlong #endif @@ -391,7 +391,7 @@ typedef unsigned char Byte; /* 8 bits */ #endif typedef unsigned int uInt; /* 16 bits or more */ -typedef unsigned long uLong; /* 32 bits or more */ +typedef uint64_t uLong; /* 32 bits or more */ #ifdef SMALL_MEDIUM /* Borland C/C++ and some old MSC versions ignore FAR inside typedef */ @@ -419,7 +419,7 @@ typedef uLong FAR uLongf; # if (UINT_MAX == 0xffffffffUL) # define Z_U4 unsigned # elif (ULONG_MAX == 0xffffffffUL) -# define Z_U4 unsigned long +# define Z_U4 uint64_t # elif (USHRT_MAX == 0xffffffffUL) # define Z_U4 unsigned short # endif @@ -428,7 +428,7 @@ typedef uLong FAR uLongf; #ifdef Z_U4 typedef Z_U4 z_crc_t; #else - typedef unsigned long z_crc_t; + typedef uint64_t z_crc_t; #endif #if 1 /* was set to #if 1 by ./configure */ @@ -501,7 +501,7 @@ typedef uLong FAR uLongf; #endif #ifndef z_off_t -# define z_off_t long +# define z_off_t int64_t #endif #if !defined(_WIN32) && defined(Z_LARGE64) diff --git a/deps/zlib-1.2.11/src/zlib.h b/deps/zlib-1.2.11/src/zlib.h index f09cdaf1e0..f407de4233 100644 --- a/deps/zlib-1.2.11/src/zlib.h +++ b/deps/zlib-1.2.11/src/zlib.h @@ -999,7 +999,7 @@ ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm, stream state was inconsistent. */ -ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm)); +ZEXTERN int64_t ZEXPORT inflateMark OF((z_streamp strm)); /* This function returns two values, one in the lower 16 bits of the return value, and the other in the remaining upper bits, obtained by shifting the @@ -1890,7 +1890,7 @@ ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp)); ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table OF((void)); ZEXTERN int ZEXPORT inflateUndermine OF((z_streamp, int)); ZEXTERN int ZEXPORT inflateValidate OF((z_streamp, int)); -ZEXTERN unsigned long ZEXPORT inflateCodesUsed OF ((z_streamp)); +ZEXTERN uint64_t ZEXPORT inflateCodesUsed OF ((z_streamp)); ZEXTERN int ZEXPORT inflateResetKeep OF((z_streamp)); ZEXTERN int ZEXPORT deflateResetKeep OF((z_streamp)); #if (defined(_WIN32) || defined(__CYGWIN__)) && !defined(Z_SOLO) diff --git a/deps/zlib-1.2.11/src/zutil.c b/deps/zlib-1.2.11/src/zutil.c index a76c6b0c7e..9883e63b26 100644 --- a/deps/zlib-1.2.11/src/zutil.c +++ b/deps/zlib-1.2.11/src/zutil.c @@ -4,7 +4,7 @@ */ /* @(#) $Id$ */ - +#include #include "zutil.h" #ifndef Z_SOLO # include "gzguts.h" @@ -280,7 +280,7 @@ void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, uInt items, uInt size) { (void)opaque; - return _halloc((long)items, size); + return _halloc((int64_t)items, size); } void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) diff --git a/deps/zlib-1.2.11/src/zutil.h b/deps/zlib-1.2.11/src/zutil.h index b079ea6a80..35dfab8c38 100644 --- a/deps/zlib-1.2.11/src/zutil.h +++ b/deps/zlib-1.2.11/src/zutil.h @@ -30,7 +30,7 @@ #endif #ifdef Z_SOLO - typedef long ptrdiff_t; /* guess -- will be caught if guess is wrong */ + typedef int64_t ptrdiff_t; /* guess -- will be caught if guess is wrong */ #endif #ifndef local @@ -44,7 +44,7 @@ typedef unsigned char uch; typedef uch FAR uchf; typedef unsigned short ush; typedef ush FAR ushf; -typedef unsigned long ulg; +typedef uint64_t ulg; extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ /* (size given to avoid silly warnings with Visual C++) */ @@ -89,7 +89,7 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # if (__STDC__ == 1) && (defined(__LARGE__) || defined(__COMPACT__)) /* Allow compilation with ANSI keywords only enabled */ void _Cdecl farfree( void *block ); - void *_Cdecl farmalloc( unsigned long nbytes ); + void *_Cdecl farmalloc( uint64_t nbytes ); # else # include # endif diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 2cd7af1dd3..1c96743381 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -131,10 +131,10 @@ # maxVnodeConnections 10000 # start http service in the cluster -# enableHttp 1 +# http 1 # start system monitor module in the cluster -# enableMonitor 1 +# monitor 1 # number of threads used to process http requests # httpMaxThreads 2 diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 9b6e851760..797a5ba824 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -1,7 +1,7 @@ #!/bin/bash # # Generate deb package for ubuntu -#set -x +# set -x #curr_dir=$(pwd) compile_dir=$1 @@ -24,7 +24,7 @@ fi mkdir -p ${pkg_dir} cd ${pkg_dir} -versioninfo=$(${script_dir}/../tools/get_version.sh) +versioninfo=$(${script_dir}/../tools/get_version.sh ${script_dir}/../../src/util/lite/src/version.c) libfile="libtaos.so.${versioninfo}" # create install dir diff --git a/packaging/deb/taosd b/packaging/deb/taosd old mode 100755 new mode 100644 diff --git a/packaging/release.sh b/packaging/release.sh index 2091bed220..a1baa76ff8 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -8,7 +8,7 @@ set -e curr_dir=$(pwd) script_dir="$(dirname $(readlink -f $0))" top_dir="$(readlink -m ${script_dir}/..)" -versioninfo="${top_dir}/src/util/src/version.c" +versioninfo="${top_dir}/src/util/lite/src/version.c" csudo="" if command -v sudo > /dev/null; then diff --git a/packaging/rpm/taosd b/packaging/rpm/taosd old mode 100755 new mode 100644 diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index c858f01d8a..1186485747 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -39,7 +39,7 @@ echo topdir: %{_topdir} echo version: %{_version} echo buildroot: %{buildroot} -versioninfo=$(%{_compiledir}/../packaging/tools/get_version.sh) +versioninfo=$(%{_compiledir}/../packaging/tools/get_version.sh ../../src/util/lite/src/version.c) libfile="libtaos.so.${versioninfo}" # create install path, and cp file diff --git a/packaging/tools/get_version.sh b/packaging/tools/get_version.sh index e0bf9d5055..0fe61e3dcb 100755 --- a/packaging/tools/get_version.sh +++ b/packaging/tools/get_version.sh @@ -7,8 +7,7 @@ set -e # set -x # -----------------------Variables definition--------------------- -script_dir=$(dirname $(readlink -m "$0")) -verinfo=$(cat ${script_dir}/../../src/util/src/version.c | grep " version" | cut -d '"' -f2) +verinfo=$(cat $1 | grep " version" | cut -d '"' -f2) verinfo=$(echo $verinfo | tr "\n" " ") len=$(echo ${#verinfo}) len=$((len-1)) diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 19e7cc4efe..ab632e6870 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -101,7 +101,7 @@ function install_lib() { # Remove links ${csudo} rm -f ${lib_link_dir}/libtaos.* || : - versioninfo=$(${script_dir}/get_version.sh) + versioninfo=$(${script_dir}/get_version.sh ${source_dir}/src/util/lite/src/version.c) ${csudo} cp ${binary_dir}/build/lib/libtaos.so.${versioninfo} ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* ${csudo} ln -sf ${install_main_dir}/driver/libtaos.so.${versioninfo} ${lib_link_dir}/libtaos.so.1 ${csudo} ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 5b880ccd3f..d7e9503e55 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -20,7 +20,7 @@ install_dir="${release_dir}/taos-${version}-${package_name}-$(echo ${build_time} # Directories and files. bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdump ${script_dir}/remove.sh" -versioninfo=$(${script_dir}/get_version.sh) +versioninfo=$(${script_dir}/get_version.sh ${code_dir}/util/lite/src/version.c) lib_files="${build_dir}/lib/libtaos.so.${versioninfo}" header_files="${code_dir}/inc/taos.h" cfg_files="${top_dir}/packaging/cfg/*.cfg" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 340a881cef..f521947770 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,9 +1,6 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -SET(PRJ_HEADER_PATH ${CMAKE_CURRENT_SOURCE_DIR}/inc) - ADD_SUBDIRECTORY(os) ADD_SUBDIRECTORY(util) ADD_SUBDIRECTORY(rpc) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt old mode 100755 new mode 100644 index fd351dc710..b43f3347ce --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -1,13 +1,14 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) +INCLUDE_DIRECTORIES(inc) +INCLUDE_DIRECTORIES(jni) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) +INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) AUX_SOURCE_DIRECTORY(./src SRC) -INCLUDE_DIRECTORIES(inc jni ${TD_ROOT_DIR}/src/inc ${TD_OS_DIR}/inc) - -IF (TD_LINUX) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/deps/jni/linux) +IF (TD_LINUX_64) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) # set the static lib name ADD_LIBRARY(taos_static STATIC ${SRC}) @@ -23,30 +24,38 @@ IF (TD_LINUX) #set version of .so #VERSION so version #SOVERSION api version - execute_process(COMMAND chmod 777 ${PROJECT_SOURCE_DIR}/../../packaging/tools/get_version.sh) - execute_process(COMMAND ${PROJECT_SOURCE_DIR}/../../packaging/tools/get_version.sh - OUTPUT_VARIABLE - VERSION_INFO) + IF (TD_LITE) + execute_process(COMMAND chmod 777 ${TD_COMMUNITY_DIR}/packaging/tools/get_version.sh) + execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_version.sh ${TD_COMMUNITY_DIR}/src/util/src/version.c + OUTPUT_VARIABLE + VERSION_INFO) + MESSAGE(STATUS "build lite version ${VERSION_INFO}") + ELSE () + execute_process(COMMAND chmod 777 ${TD_COMMUNITY_DIR}/packaging/tools/get_version.sh) + execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_version.sh ${TD_ENTERPRISE_DIR}/src/util/src/version.c + OUTPUT_VARIABLE + VERSION_INFO) + MESSAGE(STATUS "build cluster version ${VERSION_INFO}") + ENDIF () + MESSAGE(STATUS "build version ${VERSION_INFO}") SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${VERSION_INFO} SOVERSION 1) -ELSEIF (TD_WINDOWS) - - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/deps/jni/windows) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/deps/jni/windows/win32) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/deps/pthread) +ELSEIF (TD_WINDOWS_64) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows/win32) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) ADD_LIBRARY(taos_static STATIC ${SRC}) TARGET_LINK_LIBRARIES(taos_static trpc tutil) # generate dynamic library (*.dll) ADD_LIBRARY(taos SHARED ${SRC}) - SET_TARGET_PROPERTIES(taos PROPERTIES LINK_FLAGS /DEF:${TD_ROOT_DIR}/src/client/src/taos.def) + SET_TARGET_PROPERTIES(taos PROPERTIES LINK_FLAGS /DEF:${TD_COMMUNITY_DIR}/src/client/src/taos.def) TARGET_LINK_LIBRARIES(taos trpc) -ELSEIF (TD_DARWIN) - - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/deps/jni/linux) +ELSEIF (TD_DARWIN_64) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) ADD_LIBRARY(taos_static STATIC ${SRC}) TARGET_LINK_LIBRARIES(taos_static trpc tutil pthread m) @@ -55,6 +64,6 @@ ELSEIF (TD_DARWIN) # generate dynamic library (*.dylib) ADD_LIBRARY(taos SHARED ${SRC}) TARGET_LINK_LIBRARIES(taos trpc tutil pthread m) - + ENDIF () diff --git a/src/client/inc/tscJoinProcess.h b/src/client/inc/tscJoinProcess.h new file mode 100644 index 0000000000..89f29807ac --- /dev/null +++ b/src/client/inc/tscJoinProcess.h @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TSCJOINPROCESS_H +#define TDENGINE_TSCJOINPROCESS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tscUtil.h" +#include "tsclient.h" + +void tscFetchDatablockFromSubquery(SSqlObj* pSql); +void tscGetQualifiedTSList(SSqlObj* pSql, SJoinSubquerySupporter* p1, SJoinSubquerySupporter* p2, int32_t* num); + +void tscSetupOutputColumnIndex(SSqlObj* pSql); +int32_t tscLaunchSecondSubquery(SSqlObj* pSql); +void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code); + +SJoinSubquerySupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, int32_t index); +void tscDestroyJoinSupporter(SJoinSubquerySupporter* pSupporter); + +#define MEM_BUF_SIZE (1<<20) +#define TS_COMP_BLOCK_PADDING 0xFFFFFFFF +#define TS_COMP_FILE_MAGIC 0x87F5EC4C +#define TS_COMP_FILE_VNODE_MAX 512 + +typedef struct STSList { + char* rawBuf; + int32_t allocSize; + int32_t threshold; + int32_t len; +} STSList; + +typedef struct STSRawBlock { + int32_t vnode; + int64_t tag; + TSKEY* ts; + int32_t len; +} STSRawBlock; + +typedef struct STSElem { + TSKEY ts; + int64_t tag; + int32_t vnode; +} STSElem; + +typedef struct STSCursor { + int32_t vnodeIndex; + int32_t blockIndex; + int32_t tsIndex; + int32_t order; +} STSCursor; + +typedef struct STSBlock { + int64_t tag; // tag value + int32_t numOfElem; // number of elements + int32_t compLen; // size after compressed + int32_t padding; // 0xFFFFFFFF by default, after the payload + char* payload; // actual data that is compressed +} STSBlock; + +typedef struct STSVnodeBlockInfo { + int32_t vnode; + + /* + * The size of buffer file is not expected to be greater than 2G, + * and the offset of int32_t type is enough + */ + int32_t offset; + int32_t numOfBlocks; + int32_t compLen; +} STSVnodeBlockInfo; + +typedef struct STSVnodeBlockInfoEx { + STSVnodeBlockInfo info; + int32_t len; // length before compress +} STSVnodeBlockInfoEx; + +typedef struct STSBuf { + FILE* f; + char path[PATH_MAX]; + uint32_t fileSize; + + STSVnodeBlockInfoEx* pData; + int32_t numOfAlloc; + int32_t numOfVnodes; + + char* assistBuf; + int32_t bufSize; + STSBlock block; + STSList tsData; // uncompressed raw ts data + + uint64_t numOfTotal; + bool autoDelete; + int32_t tsOrder; // order of timestamp in ts comp buffer + + STSCursor cur; +} STSBuf; + +typedef struct STSBufFileHeader { + uint32_t magic; // file magic number + uint32_t numOfVnode; // number of vnode stored in current file + uint32_t tsOrder; // timestamp order in current file +} STSBufFileHeader; + +STSBuf* tsBufCreate(bool autoDelete); +STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete); +STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder); + +void tsBufDestory(STSBuf* pTSBuf); + +void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData, int32_t len); +int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeIdx); + +STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId); + +void tsBufFlush(STSBuf* pTSBuf); + +void tsBufResetPos(STSBuf* pTSBuf); +STSElem tsBufGetElem(STSBuf* pTSBuf); +bool tsBufNextPos(STSBuf* pTSBuf); + +STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag); + +STSCursor tsBufGetCursor(STSBuf* pTSBuf); +void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order); + +void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur); +STSBuf* tsBufClone(STSBuf* pTSBuf); + +/** + * display all data in comp block file, for debug purpose only + * @param pTSBuf + */ +void tsBufDisplay(STSBuf* pTSBuf); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TSCJOINPROCESS_H diff --git a/src/client/inc/tscSecondaryMerge.h b/src/client/inc/tscSecondaryMerge.h index ce3f4091d5..4c95994dfa 100644 --- a/src/client/inc/tscSecondaryMerge.h +++ b/src/client/inc/tscSecondaryMerge.h @@ -52,74 +52,53 @@ enum { }; typedef struct SLocalReducer { - SLocalDataSource **pLocalDataSrc; - int32_t numOfBuffer; - int32_t numOfCompleted; - - int32_t numOfVnode; - - SLoserTreeInfo *pLoserTree; - char * prevRowOfInput; - - tFilePage *pResultBuf; - int32_t nResultBufSize; - - char *pBufForInterpo; // intermediate buffer for interpolation - - tFilePage *pTempBuffer; - + SLocalDataSource ** pLocalDataSrc; + int32_t numOfBuffer; + int32_t numOfCompleted; + int32_t numOfVnode; + SLoserTreeInfo * pLoserTree; + char * prevRowOfInput; + tFilePage * pResultBuf; + int32_t nResultBufSize; + char * pBufForInterpo; // intermediate buffer for interpolation + tFilePage * pTempBuffer; struct SQLFunctionCtx *pCtx; - - int32_t rowSize; // size of each intermediate result. - int32_t status; // denote it is in reduce process, in reduce process, it - // cannot be released - bool hasPrevRow; - bool hasUnprocessedRow; - - tOrderDescriptor *pDesc; - tColModel * resColModel; - - tExtMemBuffer ** pExtMemBuffer; // disk-based buffer - SInterpolationInfo interpolationInfo; // interpolation support structure - - char *pFinalRes; // result data after interpo - - tFilePage *discardData; - bool discard; - - int32_t offset; + int32_t rowSize; // size of each intermediate result. + int32_t status; // denote it is in reduce process, in reduce process, it + bool hasPrevRow; // cannot be released + bool hasUnprocessedRow; + tOrderDescriptor * pDesc; + tColModel * resColModel; + tExtMemBuffer ** pExtMemBuffer; // disk-based buffer + SInterpolationInfo interpolationInfo; // interpolation support structure + char * pFinalRes; // result data after interpo + tFilePage * discardData; + SResultInfo * pResInfo; + bool discard; + int32_t offset; // limit offset value } SLocalReducer; -typedef struct SRetrieveSupport { - tExtMemBuffer ** pExtMemBuffer; // for build loser tree - tOrderDescriptor *pOrderDescriptor; - tColModel * pFinalColModel; // colModel for final result - - /* - * shared by all subqueries - * It is the number of completed retrieval subquery. - * once this value equals to numOfVnodes, all retrieval are completed. - * Local merge is launched. - */ - int32_t *numOfFinished; - int32_t numOfVnodes; // total number of vnode - int32_t vnodeIdx; // index of current vnode in vnode list - +typedef struct SSubqueryState { /* - * shared by all subqueries - * denote the status of query on vnode, if code!=0, all following - * retrieval on vnode are aborted. + * the number of completed retrieval subquery, once this value equals to numOfVnodes, + * all retrieval are completed.Local merge is launched. */ - int32_t *code; - - SSqlObj * pParentSqlObj; - tFilePage *localBuffer; // temp buffer, there is a buffer for each vnode to - // save data - uint64_t *numOfTotalRetrievedPoints; // total number of points in this query - // retrieved from server + int32_t numOfCompleted; + int32_t numOfTotal; // number of total sub-queries + int32_t code; // code from subqueries + uint64_t numOfRetrievedRows; // total number of points in this query +} SSubqueryState; - uint32_t numOfRetry; // record the number of retry times - pthread_mutex_t queryMutex; +typedef struct SRetrieveSupport { + tExtMemBuffer ** pExtMemBuffer; // for build loser tree + tOrderDescriptor *pOrderDescriptor; + tColModel * pFinalColModel; // colModel for final result + SSubqueryState * pState; + int32_t vnodeIdx; // index of current vnode in vnode list + SSqlObj * pParentSqlObj; + tFilePage * localBuffer; // temp buffer, there is a buffer for each vnode to + uint32_t numOfRetry; // record the number of retry times + pthread_mutex_t queryMutex; } SRetrieveSupport; int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOrderDescriptor **pDesc, diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index c5c634a9c5..0474697c3f 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -23,18 +23,20 @@ extern "C" { /* * @date 2018/09/30 */ +#include #include -#include "tsdb.h" -#include "tsclient.h" #include "textbuffer.h" +#include "tsclient.h" +#include "tsdb.h" +#include "tscSecondaryMerge.h" #define UTIL_METER_IS_METRIC(cmd) (((cmd)->pMeterMeta != NULL) && ((cmd)->pMeterMeta->meterType == TSDB_METER_METRIC)) - #define UTIL_METER_IS_NOMRAL_METER(cmd) (!(UTIL_METER_IS_METRIC(cmd))) - #define UTIL_METER_IS_CREATE_FROM_METRIC(cmd) \ (((cmd)->pMeterMeta != NULL) && ((cmd)->pMeterMeta->meterType == TSDB_METER_MTABLE)) +#define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) + typedef struct SParsedColElem { int16_t colIndex; int16_t offset; @@ -47,15 +49,36 @@ typedef struct SParsedDataColInfo { bool hasVal[TSDB_MAX_COLUMNS]; } SParsedDataColInfo; -STableDataBlocks* tscCreateDataBlock(int32_t size); +typedef struct SJoinSubquerySupporter { + SSubqueryState* pState; + SSqlObj* pObj; // parent SqlObj + bool hasMore; // has data from vnode to fetch + int32_t subqueryIndex; // index of sub query + int64_t interval; // interval time + SLimitVal limit; // limit info + uint64_t uid; // query meter uid + SColumnBaseInfo colList; // previous query information + SSqlExprInfo exprsInfo; + SFieldInfo fieldsInfo; + STagCond tagCond; + SSqlGroupbyExpr groupbyExpr; + + struct STSBuf* pTSBuf; + + FILE* f; + char path[PATH_MAX]; +} SJoinSubquerySupporter; + void tscDestroyDataBlock(STableDataBlocks* pDataBlock); +STableDataBlocks* tscCreateDataBlock(int32_t size); void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks); +SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, uint32_t offset); SDataBlockList* tscCreateBlockArrayList(); void* tscDestroyBlockArrayList(SDataBlockList* pList); int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock); void tscFreeUnusedDataBlocks(SDataBlockList* pList); -void tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pDataList); +int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pDataList); STableDataBlocks* tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, char* tableId); STableDataBlocks* tscCreateDataBlockEx(size_t size, int32_t rowSize, int32_t startOffset, char* name); @@ -63,9 +86,6 @@ STableDataBlocks* tscCreateDataBlockEx(size_t size, int32_t rowSize, int32_t sta SVnodeSidList* tscGetVnodeSidList(SMetricMeta* pMetricmeta, int32_t vnodeIdx); SMeterSidExtInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx); -bool tscProjectionQueryOnMetric(SSqlObj* pSql); -bool tscIsTwoStageMergeMetricQuery(SSqlObj* pSql); - /** * * for the projection query on metric or point interpolation query on metric, @@ -73,52 +93,71 @@ bool tscIsTwoStageMergeMetricQuery(SSqlObj* pSql); * * @param pSql sql object * @return - * */ -bool tscIsFirstProjQueryOnMetric(SSqlObj* pSql); bool tscIsPointInterpQuery(SSqlCmd* pCmd); -void tscClearInterpInfo(SSqlCmd* pCmd); +bool tscIsTWAQuery(SSqlCmd* pCmd); +bool tscProjectionQueryOnMetric(SSqlCmd* pCmd); +bool tscIsTwoStageMergeMetricQuery(SSqlCmd* pCmd); +bool tscQueryOnMetric(SSqlCmd* pCmd); +bool tscQueryMetricTags(SSqlCmd* pCmd); +bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd); + +void tscAddSpecialColumnForSelect(SSqlCmd* pCmd, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, + SSchema* pColSchema, int16_t isTag); -int32_t setMeterID(SSqlObj* pSql, SSQLToken* pzTableName); +void addRequiredTagColumn(SSqlCmd* pCmd, int32_t tagColIndex, int32_t tableIndex); +void SStringFree(SString* str); +void SStringCopy(SString* pDest, const SString* pSrc); +SString SStringCreate(const char* str); + +int32_t SStringAlloc(SString* pStr, int32_t size); +int32_t SStringEnsureRemain(SString* pStr, int32_t size); + +int32_t setMeterID(SSqlObj* pSql, SSQLToken* pzTableName, int32_t tableIndex); +void tscClearInterpInfo(SSqlCmd* pCmd); bool tscIsInsertOrImportData(char* sqlstr); /* use for keep current db info temporarily, for handle table with db prefix */ void tscGetDBInfoFromMeterId(char* meterId, char* db); -void tscGetMetricMetaCacheKey(SSqlCmd* pCmd, char* keyStr); -bool tscQueryOnMetric(SSqlCmd* pCmd); - -int tscAllocPayloadWithSize(SSqlCmd* pCmd, int size); +int tscAllocPayload(SSqlCmd* pCmd, int size); void tscFieldInfoSetValFromSchema(SFieldInfo* pFieldInfo, int32_t index, SSchema* pSchema); void tscFieldInfoSetValFromField(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* pField); void tscFieldInfoSetValue(SFieldInfo* pFieldInfo, int32_t index, int8_t type, char* name, int16_t bytes); +void tscFieldInfoUpdateVisible(SFieldInfo* pFieldInfo, int32_t index, bool visible); void tscFieldInfoCalOffset(SSqlCmd* pCmd); -void tscFieldInfoRenewOffsetForInterResult(SSqlCmd* pCmd); -void tscFieldInfoClone(SFieldInfo* src, SFieldInfo* dst); +void tscFieldInfoUpdateOffset(SSqlCmd* pCmd); +void tscFieldInfoCopy(SFieldInfo* src, SFieldInfo* dst, const int32_t* indexList, int32_t size); +void tscFieldInfoCopyAll(SFieldInfo* src, SFieldInfo* dst); TAOS_FIELD* tscFieldInfoGetField(SSqlCmd* pCmd, int32_t index); int16_t tscFieldInfoGetOffset(SSqlCmd* pCmd, int32_t index); int32_t tscGetResRowLength(SSqlCmd* pCmd); -void tscClearFieldInfo(SSqlCmd* pCmd); +void tscClearFieldInfo(SFieldInfo* pFieldInfo); -void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes); +void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex); -SSqlExpr* tscSqlExprInsert(SSqlCmd* pCmd, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, - int16_t size); +SSqlExpr* tscSqlExprInsert(SSqlCmd* pCmd, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, /*int16_t colId,*/ int16_t interSize); SSqlExpr* tscSqlExprUpdate(SSqlCmd* pCmd, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, int16_t size); SSqlExpr* tscSqlExprGet(SSqlCmd* pCmd, int32_t index); -void tscSqlExprClone(SSqlExprInfo* src, SSqlExprInfo* dst); +void tscSqlExprCopy(SSqlExprInfo* dst, const SSqlExprInfo* src, uint64_t uid); + +SColumnBase* tscColumnBaseInfoInsert(SSqlCmd* pCmd, SColumnIndex* colIndex); +void tscColumnFilterInfoCopy(SColumnFilterInfo* dst, const SColumnFilterInfo* src); +void tscColumnBaseCopy(SColumnBase* dst, const SColumnBase* src); -SColumnBase* tscColumnInfoInsert(SSqlCmd* pCmd, int32_t colIndex); +void tscColumnBaseInfoCopy(SColumnBaseInfo* dst, const SColumnBaseInfo* src, int16_t tableIndex); +SColumnBase* tscColumnBaseInfoGet(SColumnBaseInfo* pColumnBaseInfo, int32_t index); +void tscColumnBaseInfoUpdateTableIndex(SColumnBaseInfo* pColList, int16_t tableIndex); -void tscColumnInfoClone(SColumnsInfo* src, SColumnsInfo* dst); -SColumnBase* tscColumnInfoGet(SSqlCmd* pCmd, int32_t index); -void tscColumnInfoReserve(SSqlCmd* pCmd, int32_t size); +void tscColumnBaseInfoReserve(SColumnBaseInfo* pColumnBaseInfo, int32_t size); +void tscColumnBaseInfoDestroy(SColumnBaseInfo* pColumnBaseInfo); int32_t tscValidateName(SSQLToken* pToken); @@ -127,8 +166,10 @@ void tscIncStreamExecutionCount(void* pStream); bool tscValidateColumnId(SSqlCmd* pCmd, int32_t colId); // get starter position of metric query condition (query on tags) in SSqlCmd.payload -char* tsGetMetricQueryCondPos(STagCond* pCond); -void tscTagCondAssign(STagCond* pDst, STagCond* pSrc); +SCond* tsGetMetricQueryCondPos(STagCond* pCond, uint64_t tableIndex); +void tsSetMetricQueryCond(STagCond* pTagCond, uint64_t uid, const char* str); + +void tscTagCondCopy(STagCond* dest, const STagCond* src); void tscTagCondRelease(STagCond* pCond); void tscTagCondSetQueryCondType(STagCond* pCond, int16_t type); @@ -138,8 +179,54 @@ void tscSetFreeHeatBeat(STscObj* pObj); bool tscShouldFreeHeatBeat(SSqlObj* pHb); void tscCleanSqlCmd(SSqlCmd* pCmd); bool tscShouldFreeAsyncSqlObj(SSqlObj* pSql); + +void tscRemoveAllMeterMetaInfo(SSqlCmd* pCmd, bool removeFromCache); +SMeterMetaInfo* tscGetMeterMetaInfo(SSqlCmd* pCmd, int32_t index); +SMeterMetaInfo* tscGetMeterMetaInfoByUid(SSqlCmd* pCmd, uint64_t uid, int32_t* index); +void tscClearMeterMetaInfo(SMeterMetaInfo* pMeterMetaInfo, bool removeFromCache); + +SMeterMetaInfo* tscAddMeterMetaInfo(SSqlCmd* pCmd, const char* name, SMeterMeta* pMeterMeta, SMetricMeta* pMetricMeta, + int16_t numOfTags, int16_t* tags); +SMeterMetaInfo* tscAddEmptyMeterMetaInfo(SSqlCmd* pCmd); + +void tscGetMetricMetaCacheKey(SSqlCmd* pCmd, char* keyStr, uint64_t uid); +int tscGetMetricMeta(SSqlObj* pSql); +int tscGetMeterMeta(SSqlObj* pSql, char* meterId, int32_t tableIndex); +int tscGetMeterMetaEx(SSqlObj* pSql, char* meterId, bool createIfNotExists); + +void tscResetForNextRetrieve(SSqlRes* pRes); + +void tscAddTimestampColumn(SSqlCmd* pCmd, int16_t functionId, int16_t tableIndex); void tscDoQuery(SSqlObj* pSql); +/** + * The create object function must be successful expect for the out of memory issue. + * + * Therefore, the metermeta/metricmeta object is directly passed to the newly created subquery object from the + * previous sql object, instead of retrieving the metermeta/metricmeta from cache. + * + * Because the metermeta/metricmeta may have been released by other threads, resulting in the retrieving failed as + * well as the create function. + * + * @param pSql + * @param vnodeIndex + * @param tableIndex + * @param fp + * @param param + * @param pPrevSql + * @return + */ +SSqlObj* createSubqueryObj(SSqlObj* pSql, int32_t vnodeIndex, int16_t tableIndex, void (*fp)(), void* param, + SSqlObj* pPrevSql); +void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t tableIndex); + +void doAddGroupColumnForSubquery(SSqlCmd* pCmd, int32_t tagIndex); + +int16_t tscGetJoinTagColIndexByUid(SSqlCmd* pCmd, uint64_t uid); + +TAOS* taos_connect_a(char* ip, char* user, char* pass, char* db, int port, void (*fp)(void*, TAOS_RES*, int), + void* param, void** taos); + void sortRemoveDuplicates(STableDataBlocks* dataBuf); #ifdef __cplusplus } diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index ab5f581a0b..0d2c7dace3 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -63,9 +63,9 @@ enum _sql_cmd { TSDB_SQL_ALTER_DB, TSDB_SQL_CREATE_MNODE, TSDB_SQL_DROP_MNODE, - TSDB_SQL_CREATE_PNODE, - TSDB_SQL_DROP_PNODE, - TSDB_SQL_CFG_PNODE, // 20 + TSDB_SQL_CREATE_DNODE, + TSDB_SQL_DROP_DNODE, + TSDB_SQL_CFG_DNODE, // 20 TSDB_SQL_CFG_MNODE, TSDB_SQL_SHOW, TSDB_SQL_RETRIEVE, @@ -78,15 +78,21 @@ enum _sql_cmd { TSDB_SQL_USE_DB, TSDB_SQL_META, // 30 TSDB_SQL_METRIC, + TSDB_SQL_MULTI_META, TSDB_SQL_HB, TSDB_SQL_LOCAL, // SQL below for client local TSDB_SQL_DESCRIBE_TABLE, TSDB_SQL_RETRIEVE_METRIC, + TSDB_SQL_METRIC_JOIN_RETRIEVE, TSDB_SQL_RETRIEVE_TAGS, - TSDB_SQL_RETRIEVE_EMPTY_RESULT, // build empty result instead of accessing - // dnode to fetch result - TSDB_SQL_RESET_CACHE, // reset the client cache + /* + * build empty result instead of accessing dnode to fetch result + * reset the client cache + */ + TSDB_SQL_RETRIEVE_EMPTY_RESULT, + + TSDB_SQL_RESET_CACHE, TSDB_SQL_CFG_LOCAL, TSDB_SQL_MAX @@ -96,24 +102,35 @@ enum _sql_cmd { struct SSqlInfo; typedef struct SSqlGroupbyExpr { - int16_t numOfGroupbyCols; - int16_t tagIndex[TSDB_MAX_TAGS]; /* group by columns information */ - int16_t orderIdx; /* order by column index */ - int16_t orderType; /* order by type: asc/desc */ -} SSqlGroupbyExpr; + int16_t tableIndex; -/* the structure for sql function in select clause */ -typedef struct SSqlExpr { - char aliasName[TSDB_COL_NAME_LEN + 1]; // as aliasName + int16_t numOfGroupCols; + SColIndexEx columnInfo[TSDB_MAX_TAGS]; // group by columns information + + int16_t orderIndex; // order by column index + int16_t orderType; // order by type: asc/desc +} SSqlGroupbyExpr; - SColIndex colInfo; - int16_t sqlFuncId; // function id in aAgg array +typedef struct SMeterMetaInfo { + SMeterMeta * pMeterMeta; // metermeta + SMetricMeta *pMetricMeta; // metricmeta - int16_t resType; // return value type - int16_t resBytes; // length of return value + char name[TSDB_METER_ID_LEN + 1]; + int16_t numOfTags; // total required tags in query, including groupby tags + int16_t tagColumnIndex[TSDB_MAX_TAGS]; // clause + tag projection +} SMeterMetaInfo; - int16_t numOfParams; // argument value of each function - tVariant param[3]; // parameters are not more than 3 +/* the structure for sql function in select clause */ +typedef struct SSqlExpr { + char aliasName[TSDB_COL_NAME_LEN + 1]; // as aliasName + SColIndexEx colInfo; + int64_t uid; // refactor use the pointer + int16_t functionId; // function id in aAgg array + int16_t resType; // return value type + int16_t resBytes; // length of return value + int16_t interResBytes; // inter result buffer size + int16_t numOfParams; // argument value of each function + tVariant param[3]; // parameters are not more than 3 } SSqlExpr; typedef struct SFieldInfo { @@ -121,6 +138,15 @@ typedef struct SFieldInfo { int16_t numOfAlloc; // allocated size TAOS_FIELD *pFields; short * pOffset; + + /* + * define if this column is belong to the queried result, it may be add by parser to faciliate + * the query process + * + * NOTE: these hidden columns always locate at the end of the output columns + */ + bool * pVisibleCols; + int32_t numOfHiddenCols; // the number of column not belongs to the queried result columns } SFieldInfo; typedef struct SSqlExprInfo { @@ -129,87 +155,122 @@ typedef struct SSqlExprInfo { SSqlExpr *pExprs; } SSqlExprInfo; -typedef struct SColumnBase { - int16_t colIndex; - - /* todo refactor: the following data is belong to one struct */ - int16_t filterOn; /* denote if the filter is active */ - int16_t lowerRelOptr; - int16_t upperRelOptr; - int16_t filterOnBinary; /* denote if current column is binary */ +typedef struct SColumnIndex { + int16_t tableIndex; + int16_t columnIndex; +} SColumnIndex; - union { - struct { - int64_t lowerBndi; - int64_t upperBndi; - }; - struct { - double lowerBndd; - double upperBndd; - }; - struct { - int64_t pz; - int64_t len; - }; - }; +typedef struct SColumnBase { + SColumnIndex colIndex; + int32_t numOfFilters; + SColumnFilterInfo *filterInfo; } SColumnBase; -typedef struct SColumnsInfo { +typedef struct SColumnBaseInfo { int16_t numOfAlloc; int16_t numOfCols; SColumnBase *pColList; -} SColumnsInfo; +} SColumnBaseInfo; struct SLocalReducer; +// todo move to utility +typedef struct SString { + int32_t alloc; + int32_t n; + char * z; +} SString; + +typedef struct SCond { + uint64_t uid; + SString cond; +} SCond; + +typedef struct SJoinNode { + char meterId[TSDB_METER_ID_LEN]; + uint64_t uid; + int16_t tagCol; +} SJoinNode; + +typedef struct SJoinInfo { + bool hasJoin; + SJoinNode left; + SJoinNode right; +} SJoinInfo; + typedef struct STagCond { - int32_t len; - int32_t allocSize; - int16_t type; - char * pData; + // relation between tbname list and query condition, including : TK_AND or TK_OR + int16_t relType; + + // tbname query condition, only support tbname query condition on one table + SCond tbnameCond; + + // join condition, only support two tables join currently + SJoinInfo joinInfo; + + // for different table, the query condition must be seperated + SCond cond[TSDB_MAX_JOIN_TABLE_NUM]; + int16_t numOfTagCond; } STagCond; +typedef struct SParamInfo { + int32_t idx; + char type; + uint8_t timePrec; + short bytes; + uint32_t offset; +} SParamInfo; + typedef struct STableDataBlocks { - char meterId[TSDB_METER_ID_LEN]; - int8_t tsSource; + char meterId[TSDB_METER_ID_LEN]; + int8_t tsSource; + bool ordered; - int64_t vgid; - int64_t size; + int64_t vgid; + int64_t prevTS; - int64_t prevTS; - bool ordered; + int32_t numOfMeters; - int32_t numOfMeters; - int32_t rowSize; - uint32_t nAllocSize; + int32_t rowSize; + uint32_t nAllocSize; + uint32_t size; union { char *filename; char *pData; }; + + // for parameter ('?') binding + uint32_t numOfAllocedParams; + uint32_t numOfParams; + SParamInfo* params; } STableDataBlocks; typedef struct SDataBlockList { - int32_t idx; - int32_t nSize; - int32_t nAlloc; - char * userParam; /* user assigned parameters for async query */ - void * udfp; /* user defined function pointer, used in async model */ + int32_t idx; + int32_t nSize; + int32_t nAlloc; + char * userParam; /* user assigned parameters for async query */ + void * udfp; /* user defined function pointer, used in async model */ STableDataBlocks **pData; } SDataBlockList; typedef struct { - char name[TSDB_METER_ID_LEN]; - SOrderVal order; - int command; - int count; - int16_t isInsertFromFile; // load data from file or not - int16_t metricQuery; // metric query or not - bool existsCheck; + SOrderVal order; + int command; + + // TODO refactor + int count; + int16_t isInsertFromFile; // load data from file or not + + union { + bool existsCheck; + int8_t showType; + }; + char msgType; - char type; + uint16_t type; char intervalTimeUnit; - int64_t etime; - int64_t stime; + int64_t etime, stime; int64_t nAggTimeInterval; // aggregation time interval int64_t nSlidingTime; // sliding window in mseconds SSqlGroupbyExpr groupbyExpr; // group by tags info @@ -224,24 +285,27 @@ typedef struct { char * payload; int payloadLen; short numOfCols; - SColumnsInfo colList; + SColumnBaseInfo colList; SFieldInfo fieldsInfo; SSqlExprInfo exprsInfo; - int16_t numOfReqTags; // total required tags in query, inlcuding groupby clause + tag projection - int16_t tagColumnIndex[TSDB_MAX_TAGS + 1]; SLimitVal limit; + SLimitVal slimit; int64_t globalLimit; - SLimitVal glimit; STagCond tagCond; int16_t vnodeIdx; // vnode index in pMetricMeta for metric query int16_t interpoType; // interpolate type + int16_t numOfTables; - SDataBlockList *pDataBlocks; // submit data blocks branched according to vnode - SMeterMeta * pMeterMeta; // metermeta - SMetricMeta * pMetricMeta; // metricmeta - + // submit data blocks branched according to vnode + SDataBlockList * pDataBlocks; + SMeterMetaInfo **pMeterInfo; + struct STSBuf * tsBuf; // todo use dynamic allocated memory for defaultVal int64_t defaultVal[TSDB_MAX_COLUMNS]; // default value for interpolation + + // for parameter ('?') binding and batch processing + int32_t batchSize; + int32_t numOfParams; } SSqlCmd; typedef struct SResRec { @@ -249,28 +313,29 @@ typedef struct SResRec { int numOfTotal; } SResRec; +struct STSBuf; + typedef struct { - uint8_t code; - int numOfRows; // num of results in current retrieved - int numOfTotal; // num of total results - char * pRsp; - int rspType; - int rspLen; - uint64_t qhandle; - int64_t useconds; - int64_t offset; // offset value from vnode during projection query of stable - int row; - int16_t numOfnchar; - int16_t precision; - int32_t numOfGroups; - SResRec *pGroupRec; - char * data; - short * bytes; - void ** tsrow; - - // Buffer used to put multibytes encoded using unicode (wchar_t) - char ** buffer; + uint8_t code; + int numOfRows; // num of results in current retrieved + int numOfTotal; // num of total results + char * pRsp; + int rspType; + int rspLen; + uint64_t qhandle; + int64_t useconds; + int64_t offset; // offset value from vnode during projection query of stable + int row; + int16_t numOfnchar; + int16_t precision; + int32_t numOfGroups; + SResRec * pGroupRec; + char * data; + short * bytes; + void ** tsrow; + char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) struct SLocalReducer *pLocalReducer; + SColumnIndex * pColumnIndex; } SSqlRes; typedef struct _tsc_obj { @@ -328,7 +393,7 @@ typedef struct _sstream { int64_t num; // number of computing count /* - * bookmark the current number of result in computing, + * keep the number of current result in computing, * the value will be set to 0 before set timer for next computing */ int64_t numOfRes; @@ -336,8 +401,7 @@ typedef struct _sstream { int64_t useconds; // total elapsed time int64_t ctime; // stream created time int64_t stime; // stream next executed time - int64_t etime; // stream end query time, when time is larger then etime, the - // stream will be closed + int64_t etime; // stream end query time, when time is larger then etime, the stream will be closed int64_t interval; int64_t slidingTime; int16_t precision; @@ -346,8 +410,7 @@ typedef struct _sstream { void (*fp)(); void *param; - // Call backfunction when stream is stopped from client level - void (*callback)(void *); + void (*callback)(void *); // Callback function when stream is stopped from client level struct _sstream *prev, *next; } SSqlStream; @@ -363,14 +426,12 @@ int tsParseSql(SSqlObj *pSql, char *acct, char *db, bool multiVnodeInsertion); void tscInitMsgs(); void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle); int tscProcessSql(SSqlObj *pSql); -int tscGetMeterMeta(SSqlObj *pSql, char *meterId); -int tscGetMeterMetaEx(SSqlObj *pSql, char *meterId, bool createIfNotExists); void tscAsyncInsertMultiVnodesProxy(void *param, TAOS_RES *tres, int numOfRows); int tscRenewMeterMeta(SSqlObj *pSql, char *meterId); void tscQueueAsyncRes(SSqlObj *pSql); -int tscGetMetricMeta(SSqlObj *pSql, char *meterId); + void tscQueueAsyncError(void(*fp), void *param); int tscProcessLocalCmd(SSqlObj *pSql); @@ -381,7 +442,7 @@ int taos_retrieve(TAOS_RES *res); * transfer function for metric query in stream computing, the function need to be change * before send query message to vnode */ -void tscTansformSQLFunctionForMetricQuery(SSqlCmd *pCmd); +int32_t tscTansformSQLFunctionForMetricQuery(SSqlCmd *pCmd); void tscRestoreSQLFunctionForMetricQuery(SSqlCmd *pCmd); /** @@ -395,7 +456,7 @@ void tscClearSqlMetaInfoForce(SSqlCmd *pCmd); int32_t tscCreateResPointerInfo(SSqlCmd *pCmd, SSqlRes *pRes); void tscDestroyResPointerInfo(SSqlRes *pRes); -void tscfreeSqlCmdData(SSqlCmd *pCmd); +void tscFreeSqlCmdData(SSqlCmd *pCmd); /** * only free part of resources allocated during query. @@ -413,17 +474,6 @@ void tscFreeSqlObj(SSqlObj *pObj); void tscCloseTscObj(STscObj *pObj); -// -// support functions for async metric query. -// we declare them as global visible functions, because we need them to check if a -// failed async query in tscMeterMetaCallBack is a metric query or not. - -// expr: (fp == tscRetrieveDataRes or fp == tscRetrieveFromVnodeCallBack) -// If a query is async query, we simply abort current query process, instead of continuing -// -void tscRetrieveDataRes(void *param, TAOS_RES *tres, int numOfRows); -void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); - void tscProcessMultiVnodesInsert(SSqlObj *pSql); void tscProcessMultiVnodesInsertForFile(SSqlObj *pSql); void tscKillMetricQuery(SSqlObj *pSql); @@ -435,18 +485,18 @@ int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); void tscQueueAsyncFreeResult(SSqlObj *pSql); -extern void * pVnodeConn; -extern void * pTscMgmtConn; -extern void * tscCacheHandle; -extern uint8_t globalCode; -extern int slaveIndex; -extern void * tscTmr; -extern void * tscConnCache; -extern void * tscQhandle; -extern int tscKeepConn[]; -extern int tsInsertHeadSize; -extern int tscNumOfThreads; -extern uint32_t tsServerIp; +extern void * pVnodeConn; +extern void * pTscMgmtConn; +extern void * tscCacheHandle; +extern uint8_t globalCode; +extern int slaveIndex; +extern void * tscTmr; +extern void * tscConnCache; +extern void * tscQhandle; +extern int tscKeepConn[]; +extern int tsInsertHeadSize; +extern int tscNumOfThreads; +extern SIpStrList tscMgmtIpList; #ifdef __cplusplus } diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h old mode 100755 new mode 100644 diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c old mode 100755 new mode 100644 index 4a6b0f6d17..fb175618c0 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -19,6 +19,7 @@ #include "com_taosdata_jdbc_TSDBJNIConnector.h" #include "taos.h" #include "tlog.h" +#include "tscJoinProcess.h" #include "tsclient.h" #include "tscUtil.h" @@ -660,4 +661,4 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_validateCreateTab JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getTsCharset(JNIEnv *env, jobject jobj) { return (*env)->NewStringUTF(env, (const char *)tsCharset); -} +} \ No newline at end of file diff --git a/src/client/src/tscAst.c b/src/client/src/tscAst.c index a2475e47ca..845a9fd36d 100644 --- a/src/client/src/tscAst.c +++ b/src/client/src/tscAst.c @@ -43,14 +43,11 @@ */ static tSQLSyntaxNode *tSQLSyntaxNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken); -static void tSQLSyntaxNodeDestroy(tSQLSyntaxNode *pNode); +static void tSQLSyntaxNodeDestroy(tSQLSyntaxNode *pNode, void (*fp)(void *)); static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i); static void destroySyntaxTree(tSQLSyntaxNode *); -static void tSQLListTraversePrepare(tQueryInfo *colInfo, SSchema *pSchema, int32_t numOfCols, SSchema *pOneColSchema, - uint8_t optr, tVariant *val); - static uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tSQLSyntaxNode *pLeft, const tSQLSyntaxNode *pRight); @@ -78,9 +75,14 @@ static void reviseBinaryExprIfNecessary(tSQLSyntaxNode **pLeft, tSQLSyntaxNode * } } - // switch left and left and right hand side in expr + /* + * for expressions that are suitable for switch principle, + * switch left and left and right hand side in expr if possible + */ if ((*pLeft)->nodeType == TSQL_NODE_VALUE && (*pRight)->nodeType == TSQL_NODE_COL) { - SWAP(*pLeft, *pRight, tSQLSyntaxNode*); + if (*optr >= TSDB_RELATION_LARGE && *optr <= TSDB_RELATION_LARGE_EQUAL && *optr != TSDB_RELATION_EQUAL) { + SWAP(*pLeft, *pRight, tSQLSyntaxNode *); + } switch (*optr) { case TSDB_RELATION_LARGE: @@ -119,15 +121,14 @@ static tSQLSyntaxNode *tSQLSyntaxNodeCreate(SSchema *pSchema, int32_t numOfCols, if (strncmp(pToken->z, pSchema[i].name, pToken->n) == 0 && pToken->n == len) break; } while (++i < numOfCols); - if (i == numOfCols) { - // column name is not valid, parse the expression failed + if (i == numOfCols) { // column name is not valid, parse the expression failed return NULL; } } nodeSize += sizeof(SSchema); - pNode = malloc(nodeSize); + pNode = calloc(1, nodeSize); pNode->pSchema = (struct SSchema *)((char *)pNode + sizeof(tSQLSyntaxNode)); pNode->nodeType = TSQL_NODE_COL; @@ -144,7 +145,7 @@ static tSQLSyntaxNode *tSQLSyntaxNodeCreate(SSchema *pSchema, int32_t numOfCols, } else { nodeSize += sizeof(tVariant); - pNode = malloc(nodeSize); + pNode = calloc(1, nodeSize); pNode->pVal = (tVariant *)((char *)pNode + sizeof(tSQLSyntaxNode)); toTSDBType(pToken->type); @@ -202,12 +203,12 @@ static tSQLSyntaxNode *parseRemainStr(char *pstr, tSQLBinaryExpr *pExpr, SSchema tSQLSyntaxNode *pRight = createSyntaxTree(pSchema, numOfCols, pstr, i); if (pRight == NULL || (pRight->nodeType == TSQL_NODE_COL && pLeft->nodeType != TSQL_NODE_VALUE) || (pLeft->nodeType == TSQL_NODE_VALUE && pRight->nodeType != TSQL_NODE_COL)) { - tSQLSyntaxNodeDestroy(pLeft); - tSQLSyntaxNodeDestroy(pRight); + tSQLSyntaxNodeDestroy(pLeft, NULL); + tSQLSyntaxNodeDestroy(pRight, NULL); return NULL; } - tSQLBinaryExpr *pNewExpr = (tSQLBinaryExpr *)malloc(sizeof(tSQLBinaryExpr)); + tSQLBinaryExpr *pNewExpr = (tSQLBinaryExpr *)calloc(1, sizeof(tSQLBinaryExpr)); uint8_t k = optr; reviseBinaryExprIfNecessary(&pLeft, &pRight, &k); pNewExpr->pLeft = pLeft; @@ -228,8 +229,7 @@ uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tSQLSyntaxNode // if left node is the primary column,return true return (strcmp(primaryColumnName, pLeft->pSchema->name) == 0) ? 1 : 0; } else { - // if any children have query on primary key, their parents are also keep - // this value + // if any children have query on primary key, their parents are also keep this value return ((pLeft->nodeType == TSQL_NODE_EXPR && pLeft->pExpr->filterOnPrimaryKey == 1) || (pRight->nodeType == TSQL_NODE_EXPR && pRight->pExpr->filterOnPrimaryKey == 1)) == true ? 1 @@ -238,9 +238,9 @@ uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tSQLSyntaxNode } static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) { - SSQLToken t0 = {0}; + SSQLToken t0; - tStrGetToken(str, i, &t0, false); + t0 = tStrGetToken(str, i, false, 0, NULL); if (t0.n == 0) { return NULL; } @@ -259,11 +259,11 @@ static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, cha return NULL; } - tStrGetToken(str, i, &t0, false); + t0 = tStrGetToken(str, i, false, 0, NULL); if (t0.n == 0 || t0.type == TK_RP) { if (pLeft->nodeType != TSQL_NODE_EXPR) { // if left is not the expr, it is not a legal expr - tSQLSyntaxNodeDestroy(pLeft); + tSQLSyntaxNodeDestroy(pLeft, NULL); return NULL; } @@ -274,7 +274,7 @@ static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, cha uint8_t optr = getBinaryExprOptr(&t0); if (optr <= 0) { pError("not support binary operator:%d", t0.type); - tSQLSyntaxNodeDestroy(pLeft); + tSQLSyntaxNodeDestroy(pLeft, NULL); return NULL; } @@ -290,9 +290,9 @@ static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, cha * if we do not get the information, in case of value of field presented first, * we revised the value after the binary expression is completed. */ - tStrGetToken(str, i, &t0, true); + t0 = tStrGetToken(str, i, true, 0, NULL); if (t0.n == 0) { - tSQLSyntaxNodeDestroy(pLeft); // illegal expression + tSQLSyntaxNodeDestroy(pLeft, NULL); // illegal expression return NULL; } @@ -304,12 +304,12 @@ static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, cha } if (pRight == NULL) { - tSQLSyntaxNodeDestroy(pLeft); + tSQLSyntaxNodeDestroy(pLeft, NULL); return NULL; } /* create binary expr as the child of new parent node */ - tSQLBinaryExpr *pBinExpr = (tSQLBinaryExpr *)malloc(sizeof(tSQLBinaryExpr)); + tSQLBinaryExpr *pBinExpr = (tSQLBinaryExpr *)calloc(1, sizeof(tSQLBinaryExpr)); reviseBinaryExprIfNecessary(&pLeft, &pRight, &optr); pBinExpr->filterOnPrimaryKey = isQueryOnPrimaryKey(pSchema[0].name, pLeft, pRight); @@ -317,7 +317,7 @@ static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, cha pBinExpr->pRight = pRight; pBinExpr->nSQLBinaryOptr = optr; - tStrGetToken(str, i, &t0, true); + t0 = tStrGetToken(str, i, true, 0, NULL); if (t0.n == 0 || t0.type == TK_RP) { tSQLSyntaxNode *pn = malloc(sizeof(tSQLSyntaxNode)); @@ -366,6 +366,7 @@ int32_t tSQLBinaryExprToStringImpl(tSQLSyntaxNode *pNode, char *dst, uint8_t typ return len; } +// TODO REFACTOR WITH SQL PARSER static char *tSQLOptrToString(uint8_t optr, char *dst) { switch (optr) { case TSDB_RELATION_LESS: { @@ -432,13 +433,15 @@ void tSQLBinaryExprToString(tSQLBinaryExpr *pExpr, char *dst, int32_t *len) { *len += tSQLBinaryExprToStringImpl(pExpr->pRight, start, pExpr->pRight->nodeType); } -static void UNUSED_FUNC destroySyntaxTree(tSQLSyntaxNode *pNode) { tSQLSyntaxNodeDestroy(pNode); } +static void UNUSED_FUNC destroySyntaxTree(tSQLSyntaxNode *pNode) { tSQLSyntaxNodeDestroy(pNode, NULL); } -static void tSQLSyntaxNodeDestroy(tSQLSyntaxNode *pNode) { - if (pNode == NULL) return; +static void tSQLSyntaxNodeDestroy(tSQLSyntaxNode *pNode, void (*fp)(void *)) { + if (pNode == NULL) { + return; + } if (pNode->nodeType == TSQL_NODE_EXPR) { - tSQLBinaryExprDestroy(&pNode->pExpr); + tSQLBinaryExprDestroy(&pNode->pExpr, fp); } else if (pNode->nodeType == TSQL_NODE_VALUE) { tVariantDestroy(pNode->pVal); } @@ -446,139 +449,20 @@ static void tSQLSyntaxNodeDestroy(tSQLSyntaxNode *pNode) { free(pNode); } -void tSQLBinaryExprDestroy(tSQLBinaryExpr **pExprs) { - if (*pExprs == NULL) return; - - tSQLSyntaxNodeDestroy((*pExprs)->pLeft); - tSQLSyntaxNodeDestroy((*pExprs)->pRight); - - free(*pExprs); - *pExprs = NULL; -} - -static int32_t compareIntVal(const void *pLeft, const void *pRight) { - DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_INT64_VAL(pRight)); -} - -static int32_t compareIntDoubleVal(const void *pLeft, const void *pRight) { - DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_DOUBLE_VAL(pRight)); -} - -static int32_t compareDoubleVal(const void *pLeft, const void *pRight) { - DEFAULT_COMP(GET_DOUBLE_VAL(pLeft), GET_DOUBLE_VAL(pRight)); -} - -static int32_t compareDoubleIntVal(const void *pLeft, const void *pRight) { - double ret = (*(double *)pLeft) - (*(int64_t *)pRight); - if (fabs(ret) < DBL_EPSILON) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -static int32_t compareStrVal(const void *pLeft, const void *pRight) { - int32_t ret = strcmp(pLeft, pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -static int32_t compareWStrVal(const void *pLeft, const void *pRight) { - int32_t ret = wcscmp(pLeft, pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -static int32_t compareStrPatternComp(const void *pLeft, const void *pRight) { - SPatternCompareInfo pInfo = {'%', '_'}; - - const char *pattern = pRight; - const char *str = pLeft; - - if (patternMatch(pattern, str, strlen(str), &pInfo) == TSDB_PATTERN_MATCH) { - return 0; - } else { - return 1; - } -} - -static int32_t compareWStrPatternComp(const void *pLeft, const void *pRight) { - SPatternCompareInfo pInfo = {'%', '_'}; - - const wchar_t *pattern = pRight; - const wchar_t *str = pLeft; - - if (WCSPatternMatch(pattern, str, wcslen(str), &pInfo) == TSDB_PATTERN_MATCH) { - return 0; - } else { - return 1; +void tSQLBinaryExprDestroy(tSQLBinaryExpr **pExpr, void (*fp)(void *)) { + if (*pExpr == NULL) { + return; } -} - -static __compar_fn_t getFilterComparator(int32_t type, int32_t filterType, int32_t optr) { - __compar_fn_t comparator = NULL; - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_BOOL: { - if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { - comparator = compareIntVal; - } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { - comparator = compareIntDoubleVal; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: { - if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { - comparator = compareDoubleIntVal; - } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { - comparator = compareDoubleVal; - } - break; - } - - case TSDB_DATA_TYPE_BINARY: { - assert(filterType == TSDB_DATA_TYPE_BINARY); + tSQLSyntaxNodeDestroy((*pExpr)->pLeft, fp); + tSQLSyntaxNodeDestroy((*pExpr)->pRight, fp); - if (optr == TSDB_RELATION_LIKE) { - /* wildcard query using like operator */ - comparator = compareStrPatternComp; - } else { - /* normal relational comparator */ - comparator = compareStrVal; - } - - break; - } - - case TSDB_DATA_TYPE_NCHAR: { - assert(filterType == TSDB_DATA_TYPE_NCHAR); - - if (optr == TSDB_RELATION_LIKE) { - comparator = compareWStrPatternComp; - } else { - comparator = compareWStrVal; - } - - break; - } - default: - comparator = compareIntVal; - break; + if (fp != NULL) { + fp((*pExpr)->info); } - return comparator; + free(*pExpr); + *pExpr = NULL; } static void setInitialValueForRangeQueryCondition(tSKipListQueryCond *q, int8_t type) { @@ -619,104 +503,59 @@ static void setInitialValueForRangeQueryCondition(tSKipListQueryCond *q, int8_t } } -static void tSQLDoFilterInitialResult(tSkipList *pSkipList, bool (*fp)(), tQueryInfo *colInfo, +static void tSQLDoFilterInitialResult(tSkipList *pSkipList, bool (*fp)(), tQueryInfo *queryColInfo, tQueryResultset *result) { - // primary key, search according to skiplist - if (colInfo->colIdx == 0 && colInfo->optr != TSDB_RELATION_LIKE) { + // primary key filter, search according to skiplist + if (queryColInfo->colIdx == 0 && queryColInfo->optr != TSDB_RELATION_LIKE) { tSKipListQueryCond q; - setInitialValueForRangeQueryCondition(&q, colInfo->q.nType); + setInitialValueForRangeQueryCondition(&q, queryColInfo->q.nType); - switch (colInfo->optr) { + switch (queryColInfo->optr) { case TSDB_RELATION_EQUAL: { result->num = - tSkipListPointQuery(pSkipList, &colInfo->q, 1, INCLUDE_POINT_QUERY, (tSkipListNode ***)&result->pRes); + tSkipListPointQuery(pSkipList, &queryColInfo->q, 1, INCLUDE_POINT_QUERY, (tSkipListNode ***)&result->pRes); break; } case TSDB_RELATION_NOT_EQUAL: { result->num = - tSkipListPointQuery(pSkipList, &colInfo->q, 1, EXCLUDE_POINT_QUERY, (tSkipListNode ***)&result->pRes); + tSkipListPointQuery(pSkipList, &queryColInfo->q, 1, EXCLUDE_POINT_QUERY, (tSkipListNode ***)&result->pRes); break; } case TSDB_RELATION_LESS_EQUAL: { - tVariantAssign(&q.upperBnd, &colInfo->q); - q.upperBndRelOptr = colInfo->optr; + tVariantAssign(&q.upperBnd, &queryColInfo->q); + q.upperBndRelOptr = queryColInfo->optr; result->num = tSkipListQuery(pSkipList, &q, (tSkipListNode ***)&result->pRes); break; } case TSDB_RELATION_LESS: { - tVariantAssign(&q.upperBnd, &colInfo->q); + tVariantAssign(&q.upperBnd, &queryColInfo->q); result->num = tSkipListQuery(pSkipList, &q, (tSkipListNode ***)&result->pRes); break; } case TSDB_RELATION_LARGE: { - tVariantAssign(&q.lowerBnd, &colInfo->q); + tVariantAssign(&q.lowerBnd, &queryColInfo->q); result->num = tSkipListQuery(pSkipList, &q, (tSkipListNode ***)&result->pRes); break; } case TSDB_RELATION_LARGE_EQUAL: { - tVariantAssign(&q.lowerBnd, &colInfo->q); - q.lowerBndRelOptr = colInfo->optr; + tVariantAssign(&q.lowerBnd, &queryColInfo->q); + q.lowerBndRelOptr = queryColInfo->optr; result->num = tSkipListQuery(pSkipList, &q, (tSkipListNode ***)&result->pRes); break; } default: - pTrace("skiplist:%p, unsupport query operator:%d", pSkipList, colInfo->optr); + pTrace("skiplist:%p, unsupport query operator:%d", pSkipList, queryColInfo->optr); } tSkipListDestroyKey(&q.upperBnd); tSkipListDestroyKey(&q.lowerBnd); } else { - // brutal force search - result->num = tSkipListIterateList(pSkipList, (tSkipListNode ***)&result->pRes, fp, colInfo); - } -} - -void tSQLListTraversePrepare(tQueryInfo *colInfo, SSchema *pSchema, int32_t numOfCols, SSchema *pOneColSchema, - uint8_t optr, tVariant *val) { - int32_t i = 0, offset = 0; - if (strcasecmp(pOneColSchema->name, TSQL_TBNAME_L) == 0) { - i = -1; - offset = -1; - } else { - while (i < numOfCols) { - if (pSchema[i].bytes == pOneColSchema->bytes && pSchema[i].type == pOneColSchema->type && - strcmp(pSchema[i].name, pOneColSchema->name) == 0) { - break; - } else { - offset += pSchema[i++].bytes; - } - } - } - - colInfo->pSchema = pSchema; - colInfo->colIdx = i; - colInfo->optr = optr; - colInfo->offset = offset; - colInfo->comparator = getFilterComparator(pOneColSchema->type, val->nType, optr); - - if (colInfo->pSchema[i].type != val->nType) { - /* convert the query string to be inline with the data type of the queried tags */ - if (colInfo->pSchema[i].type == TSDB_DATA_TYPE_NCHAR && val->nType == TSDB_DATA_TYPE_BINARY) { - colInfo->q.nLen = TSDB_MAX_TAGS_LEN / TSDB_NCHAR_SIZE; - - colInfo->q.wpz = calloc(1, TSDB_MAX_TAGS_LEN); - colInfo->q.nType = TSDB_DATA_TYPE_NCHAR; - - taosMbsToUcs4(val->pz, val->nLen, (char *)colInfo->q.wpz, TSDB_MAX_TAGS_LEN); - colInfo->q.nLen = wcslen(colInfo->q.wpz) + 1; - return; - } else if (colInfo->pSchema[i].type == TSDB_DATA_TYPE_BINARY && val->nType == TSDB_DATA_TYPE_NCHAR) { - colInfo->q.nLen = TSDB_MAX_TAGS_LEN; - colInfo->q.pz = calloc(1, TSDB_MAX_TAGS_LEN); - colInfo->q.nType = TSDB_DATA_TYPE_BINARY; - - taosUcs4ToMbs(val->wpz, val->nLen, colInfo->q.pz); - colInfo->q.nLen = strlen(colInfo->q.pz) + 1; - return; - } + /* + * Brutal force scan the whole skiplit to find the appropriate result, + * since the filter is not applied to indexed column. + */ + result->num = tSkipListIterateList(pSkipList, (tSkipListNode ***)&result->pRes, fp, queryColInfo); } - - tVariantAssign(&colInfo->q, val); } /* @@ -731,7 +570,9 @@ static int32_t compareByAddr(const void *pLeft, const void *pRight) { } int32_t merge(tQueryResultset *pLeft, tQueryResultset *pRight, tQueryResultset *pFinalRes) { - pFinalRes->pRes = malloc(POINTER_BYTES * (pLeft->num + pRight->num)); + assert(pFinalRes->pRes == 0); + + pFinalRes->pRes = calloc((size_t)(pLeft->num + pRight->num), POINTER_BYTES); pFinalRes->num = 0; // sort according to address @@ -742,6 +583,7 @@ int32_t merge(tQueryResultset *pLeft, tQueryResultset *pRight, tQueryResultset * qsort(pRightNodes, pRight->num, sizeof(pRight->pRes[0]), compareByAddr); int32_t i = 0, j = 0; + // merge two sorted arrays in O(n) time while (i < pLeft->num && j < pRight->num) { int64_t ret = (int64_t)pLeftNodes[i] - (int64_t)pRightNodes[j]; @@ -770,7 +612,9 @@ int32_t merge(tQueryResultset *pLeft, tQueryResultset *pRight, tQueryResultset * int32_t intersect(tQueryResultset *pLeft, tQueryResultset *pRight, tQueryResultset *pFinalRes) { int64_t num = MIN(pLeft->num, pRight->num); - pFinalRes->pRes = malloc(POINTER_BYTES * num); + assert(pFinalRes->pRes == 0); + + pFinalRes->pRes = calloc(num, POINTER_BYTES); pFinalRes->num = 0; // sort according to address @@ -802,14 +646,14 @@ int32_t intersect(tQueryResultset *pLeft, tQueryResultset *pRight, tQueryResults /* * */ -void tSQLListTraverseOnResult(struct tSQLBinaryExpr *pExpr, bool (*fp)(tSkipListNode *, void *), tQueryInfo *colInfo, - tQueryResultset *pResult) { +void tSQLListTraverseOnResult(struct tSQLBinaryExpr *pExpr, bool (*fp)(tSkipListNode *, void *), + tQueryResultset * pResult) { assert(pExpr->pLeft->nodeType == TSQL_NODE_COL && pExpr->pRight->nodeType == TSQL_NODE_VALUE); // brutal force search - int32_t num = pResult->num; + int64_t num = pResult->num; for (int32_t i = 0, j = 0; i < pResult->num; ++i) { - if (fp == NULL || (fp != NULL && fp(pResult->pRes[i], colInfo) == true)) { + if (fp == NULL || (fp != NULL && fp(pResult->pRes[i], pExpr->info) == true)) { pResult->pRes[j++] = pResult->pRes[i]; } else { num--; @@ -819,32 +663,118 @@ void tSQLListTraverseOnResult(struct tSQLBinaryExpr *pExpr, bool (*fp)(tSkipList pResult->num = num; } +static bool filterItem(tSQLBinaryExpr *pExpr, const void *pItem, SBinaryFilterSupp *param) { + tSQLSyntaxNode *pLeft = pExpr->pLeft; + tSQLSyntaxNode *pRight = pExpr->pRight; + + /* + * non-leaf nodes, recursively traverse the syntax tree in the post-root order + */ + if (pLeft->nodeType == TSQL_NODE_EXPR && pRight->nodeType == TSQL_NODE_EXPR) { + if (pExpr->nSQLBinaryOptr == TSDB_RELATION_OR) { // or + if (filterItem(pLeft->pExpr, pItem, param)) { + return true; + } + + // left child does not satisfy the query condition, try right child + return filterItem(pRight->pExpr, pItem, param); + } else { // and + if (!filterItem(pLeft->pExpr, pItem, param)) { + return false; + } + + return filterItem(pRight->pExpr, pItem, param); + } + } + + // handle the leaf node + assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); + param->setupInfoFn(pExpr, param->pExtInfo); + + return param->fp(pItem, pExpr->info); +} + +/** + * Apply the filter expression on non-indexed tag columns to each element in the result list, which is generated + * by filtering on indexed tag column. So the whole result set only needs to be iterated once to generate + * result that is satisfied to the filter expression, no matter how the filter expression consisting of. + * + * @param pExpr filter expression on non-indexed tag columns. + * @param pResult results from filter on the indexed tag column, which is usually the first tag column + * @param pSchema tag schemas + * @param fp filter callback function + */ +static void tSQLBinaryTraverseOnResult(tSQLBinaryExpr *pExpr, tQueryResultset *pResult, SBinaryFilterSupp *param) { + int32_t n = 0; + for (int32_t i = 0; i < pResult->num; ++i) { + void *pItem = pResult->pRes[i]; + + if (filterItem(pExpr, pItem, param)) { + pResult->pRes[n++] = pResult->pRes[i]; + } + } + + pResult->num = n; +} + +static void tSQLBinaryTraverseOnSkipList(tSQLBinaryExpr *pExpr, tQueryResultset *pResult, tSkipList *pSkipList, + SBinaryFilterSupp *param) { + int32_t n = 0; + SSkipListIterator iter = {0}; + + int32_t ret = tSkipListIteratorReset(pSkipList, &iter); + assert(ret == 0); + + pResult->pRes = calloc(pSkipList->nSize, POINTER_BYTES); + + while (tSkipListIteratorNext(&iter)) { + tSkipListNode *pNode = tSkipListIteratorGet(&iter); + if (filterItem(pExpr, pNode, param)) { + pResult->pRes[n++] = pNode; + } + } + + pResult->num = n; +} + // post-root order traverse syntax tree -void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExprs, tSkipList *pSkipList, SSchema *pSchema, int32_t numOfCols, - bool (*fp)(tSkipListNode *, void *), tQueryResultset *result) { - if (pExprs == NULL) return; +void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, tSkipList *pSkipList, tQueryResultset *result, + SBinaryFilterSupp *param) { + if (pExpr == NULL) { + return; + } - tSQLSyntaxNode *pLeft = pExprs->pLeft; - tSQLSyntaxNode *pRight = pExprs->pRight; + tSQLSyntaxNode *pLeft = pExpr->pLeft; + tSQLSyntaxNode *pRight = pExpr->pRight; // recursive traverse left child branch if (pLeft->nodeType == TSQL_NODE_EXPR || pRight->nodeType == TSQL_NODE_EXPR) { uint8_t weight = pLeft->pExpr->filterOnPrimaryKey + pRight->pExpr->filterOnPrimaryKey; if (weight == 0 && result->num > 0 && pSkipList == NULL) { - /* base on the initial filter result to perform the secondary filter */ - tSQLBinaryExprTraverse(pLeft->pExpr, pSkipList, pSchema, numOfCols, fp, result); - tSQLBinaryExprTraverse(pRight->pExpr, pSkipList, pSchema, numOfCols, fp, result); - } else if (weight == 0 || weight == 2 || (weight == 1 && pExprs->nSQLBinaryOptr == TSDB_RELATION_OR)) { + /** + * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. + * Since no index presented, the filter operation is done by scan all elements in the result set. + * + * if the query is a high selectivity filter, only small portion of meters are retrieved. + */ + tSQLBinaryTraverseOnResult(pExpr, result, param); + } else if (weight == 0) { + /** + * apply the hierarchical expression to every node in skiplist for find the qualified nodes + */ + assert(result->num == 0); + tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); + } else if (weight == 2 || (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_OR)) { tQueryResultset rLeft = {0}; tQueryResultset rRight = {0}; - tSQLBinaryExprTraverse(pLeft->pExpr, pSkipList, pSchema, numOfCols, fp, &rLeft); - tSQLBinaryExprTraverse(pRight->pExpr, pSkipList, pSchema, numOfCols, fp, &rRight); + tSQLBinaryExprTraverse(pLeft->pExpr, pSkipList, &rLeft, param); + tSQLBinaryExprTraverse(pRight->pExpr, pSkipList, &rRight, param); - if (pExprs->nSQLBinaryOptr == TSDB_RELATION_AND) { // CROSS + if (pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) { // CROSS intersect(&rLeft, &rRight, result); - } else if (pExprs->nSQLBinaryOptr == TSDB_RELATION_OR) { // or + } else if (pExpr->nSQLBinaryOptr == TSDB_RELATION_OR) { // or merge(&rLeft, &rRight, result); } else { assert(false); @@ -854,43 +784,44 @@ void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExprs, tSkipList *pSkipList, SSchem free(rRight.pRes); } else { /* - * first, we filter results based on the skiplist index, which is initial filter stage, + * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here + * + * first, we filter results based on the skiplist index, which is the initial filter stage, * then, we conduct the secondary filter operation based on the result from the initial filter stage. */ - if (pExprs->nSQLBinaryOptr == TSDB_RELATION_AND) { - tSQLBinaryExpr *pFirst = (pLeft->pExpr->filterOnPrimaryKey == 1) ? pLeft->pExpr : pRight->pExpr; - tSQLBinaryExpr *pSec = (pLeft->pExpr->filterOnPrimaryKey == 1) ? pRight->pExpr : pLeft->pExpr; - assert(pFirst != pSec && pFirst != NULL && pSec != NULL); - - // we filter the result based on the skiplist index - tSQLBinaryExprTraverse(pFirst, pSkipList, pSchema, numOfCols, fp, result); - - /* - * recursively perform the filter operation based on the initial results, - * So, we do not set the skiplist index as a parameter - */ - tSQLBinaryExprTraverse(pSec, NULL, pSchema, numOfCols, fp, result); + assert(pExpr->nSQLBinaryOptr == TSDB_RELATION_AND); + + tSQLBinaryExpr *pFirst = NULL; + tSQLBinaryExpr *pSecond = NULL; + if (pLeft->pExpr->filterOnPrimaryKey == 1) { + pFirst = pLeft->pExpr; + pSecond = pRight->pExpr; } else { - assert(false); + pFirst = pRight->pExpr; + pSecond = pLeft->pExpr; } - } + assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); + + // we filter the result based on the skiplist index in the first place + tSQLBinaryExprTraverse(pFirst, pSkipList, result, param); + + /* + * recursively perform the filter operation based on the initial results, + * So, we do not set the skiplist index as a parameter + */ + tSQLBinaryExprTraverse(pSecond, NULL, result, param); + } } else { // column project assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); - tVariant *pCond = pRight->pVal; - SSchema * pTagSchema = pLeft->pSchema; - - tQueryInfo queryColInfo = {0}; - tSQLListTraversePrepare(&queryColInfo, pSchema, numOfCols, pTagSchema, pExprs->nSQLBinaryOptr, pCond); + param->setupInfoFn(pExpr, param->pExtInfo); if (pSkipList == NULL) { - tSQLListTraverseOnResult(pExprs, fp, &queryColInfo, result); + tSQLListTraverseOnResult(pExpr, param->fp, result); } else { assert(result->num == 0); - tSQLDoFilterInitialResult(pSkipList, fp, &queryColInfo, result); + tSQLDoFilterInitialResult(pSkipList, param->fp, pExpr->info, result); } - - tVariantDestroy(&queryColInfo.q); } } @@ -916,9 +847,11 @@ void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char } if (pLeft->nodeType == TSQL_NODE_EXPR) { - if (pRight->nodeType == TSQL_NODE_EXPR) { // exprLeft + exprRight - /* the type of returned value of one expression is always double float - * precious */ + if (pRight->nodeType == TSQL_NODE_EXPR) { + /* + * exprLeft + exprRight + * the type of returned value of one expression is always double float precious + */ _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, pExprs->nSQLBinaryOptr); fp(pLeftOutput, pRightOutput, numOfRows, numOfRows, pOutput, order); @@ -958,7 +891,7 @@ void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char } else if (pRight->nodeType == TSQL_NODE_COL) { // 12 + columnRight // column data specified on right-hand-side - char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->colId); + char * pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->colId); _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pSchema->type, pExprs->nSQLBinaryOptr); fp(&pLeft->pVal->i64Key, pRightInputData, 1, numOfRows, pOutput, order); @@ -994,4 +927,13 @@ void tSQLBinaryExprTrv(tSQLBinaryExpr *pExprs, int32_t *val, int16_t *ids) { ids[*val] = pRight->pSchema->colId; (*val) += 1; } +} + +void tQueryResultClean(tQueryResultset *pRes) { + if (pRes == NULL) { + return; + } + + tfree(pRes->pRes); + pRes->num = 0; } \ No newline at end of file diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index a25e00979f..6e0e1ae7c5 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -40,7 +40,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo */ static void tscProcessAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows); -void taos_query_a(TAOS *taos, char *sqlstr, void (*fp)(void *, TAOS_RES *, int), void *param) { +void taos_query_a(TAOS *taos, const char *sqlstr, void (*fp)(void *, TAOS_RES *, int), void *param) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { tscError("bug!!! pObj:%p", pObj); @@ -72,13 +72,14 @@ void taos_query_a(TAOS *taos, char *sqlstr, void (*fp)(void *, TAOS_RES *, int), pSql->fp = fp; pSql->param = param; - if (tscAllocPayloadWithSize(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) { - tscError("%p failed to alloc payload", pSql); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { + tscError("failed to malloc payload"); + tfree(pSql); tscQueueAsyncError(fp, param); - free(pSql); return; } + pSql->sqlstr = malloc(sqlLen + 1); if (pSql->sqlstr == NULL) { tscError("%p failed to malloc sql string buffer", pSql); @@ -116,9 +117,9 @@ static void tscProcessAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOf SSqlCmd *pCmd = &pSql->cmd; // sequentially retrieve data from remain vnodes first, query vnode specified by vnodeIdx - if (numOfRows == 0 && tscProjectionQueryOnMetric(pSql)) { + if (numOfRows == 0 && tscProjectionQueryOnMetric(pCmd)) { // vnode is denoted by vnodeIdx, continue to query vnode specified by vnodeIdx - assert(pCmd->vnodeIdx >= 1); + assert(pCmd->vnodeIdx >= 0); /* reach the maximum number of output rows, abort */ if (pCmd->globalLimit > 0 && pRes->numOfTotal >= pCmd->globalLimit) { @@ -128,14 +129,14 @@ static void tscProcessAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOf /* update the limit value according to current retrieval results */ pCmd->limit.limit = pCmd->globalLimit - pRes->numOfTotal; + pCmd->limit.offset = pRes->offset; - if ((++(pSql->cmd.vnodeIdx)) <= pCmd->pMetricMeta->numOfVnodes) { - pCmd->command = TSDB_SQL_SELECT; // reset flag to launch query first. + if ((++(pCmd->vnodeIdx)) < tscGetMeterMetaInfo(pCmd, 0)->pMetricMeta->numOfVnodes) { + tscTrace("%p retrieve data from next vnode:%d", pSql, pCmd->vnodeIdx); - pRes->row = 0; - pRes->numOfRows = 0; - pCmd->type = 0; + pSql->cmd.command = TSDB_SQL_SELECT; // reset flag to launch query first. + tscResetForNextRetrieve(pRes); pSql->fp = tscProcessAsyncRetrieveNextVnode; tscProcessSql(pSql); return; @@ -213,14 +214,12 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi pSql->fp = tscProcessAsyncFetchRowsProxy; pSql->param = param; - - pRes->row = 0; - pRes->numOfRows = 0; - pCmd->type = 0; + tscResetForNextRetrieve(pRes); if (pCmd->command != TSDB_SQL_RETRIEVE_METRIC && pCmd->command < TSDB_SQL_LOCAL) { pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; } + tscProcessSql(pSql); } @@ -246,9 +245,7 @@ void taos_fetch_row_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, TAOS_ROW), pSql->param = param; if (pRes->row >= pRes->numOfRows) { - pRes->row = 0; - pRes->numOfRows = 0; - pCmd->type = 0; + tscResetForNextRetrieve(pRes); pSql->fp = tscProcessAsyncRetrieve; pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; tscProcessSql(pSql); @@ -269,7 +266,7 @@ void tscProcessAsyncRetrieve(void *param, TAOS_RES *tres, int numOfRows) { if (numOfRows == 0) { // sequentially retrieve data from remain vnodes. - if (tscProjectionQueryOnMetric(pSql)) { + if (tscProjectionQueryOnMetric(pCmd)) { /* * vnode is denoted by vnodeIdx, continue to query vnode specified by vnodeIdx till all vnode have been retrieved */ @@ -284,13 +281,10 @@ void tscProcessAsyncRetrieve(void *param, TAOS_RES *tres, int numOfRows) { /* update the limit value according to current retrieval results */ pCmd->limit.limit = pCmd->globalLimit - pRes->numOfTotal; - if ((++pCmd->vnodeIdx) <= pCmd->pMetricMeta->numOfVnodes) { - pCmd->command = TSDB_SQL_SELECT; // reset flag to launch query first. - - pRes->row = 0; - pRes->numOfRows = 0; - pCmd->type = 0; + if ((++pCmd->vnodeIdx) <= tscGetMeterMetaInfo(pCmd, 0)->pMetricMeta->numOfVnodes) { + pSql->cmd.command = TSDB_SQL_SELECT; // reset flag to launch query first. + tscResetForNextRetrieve(pRes); pSql->fp = tscProcessAsyncContinueRetrieve; tscProcessSql(pSql); return; @@ -463,10 +457,10 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) { tsem_post(&pSql->rspSem); } else { tscTrace("%p renew meterMeta successfully, command:%d, code:%d, thandle:%p, retry:%d", - pSql, pSql->cmd.command, pSql->res.code, pSql->thandle, pSql->retry); + pSql, pSql->cmd.command, pSql->res.code, pSql->thandle, pSql->retry); - assert(pSql->cmd.pMeterMeta == NULL); - tscGetMeterMeta(pSql, pSql->cmd.name); + assert(tscGetMeterMetaInfo(&pSql->cmd, 0)->pMeterMeta == NULL); + tscGetMeterMeta(pSql, tscGetMeterMetaInfo(&pSql->cmd, 0)->name, 0); code = tscSendMsgToServer(pSql); if (code != 0) { @@ -486,21 +480,23 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) { if (pSql->pStream == NULL) { // check if it is a sub-query of metric query first, if true, enter another routine - // todo refactor - if (pSql->fp == tscRetrieveDataRes || pSql->fp == tscRetrieveFromVnodeCallBack) { - assert(pCmd->pMeterMeta->numOfTags != 0 && pCmd->vnodeIdx > 0 && pSql->param != NULL); + if ((pSql->cmd.type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) == TSDB_QUERY_TYPE_STABLE_SUBQUERY) { + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + assert(pMeterMetaInfo->pMeterMeta->numOfTags != 0 && pCmd->vnodeIdx >= 0 && pSql->param != NULL); SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param; SSqlObj * pParObj = trs->pParentSqlObj; - assert(pParObj->signature == pParObj && trs->vnodeIdx == pCmd->vnodeIdx && pSql->cmd.pMeterMeta->numOfTags != 0); + assert(pParObj->signature == pParObj && trs->vnodeIdx == pCmd->vnodeIdx && + pMeterMetaInfo->pMeterMeta->numOfTags != 0); + tscTrace("%p get metricMeta during metric query successfully", pSql); - code = tscGetMeterMeta(pSql, pSql->cmd.name); + code = tscGetMeterMeta(pSql, tscGetMeterMetaInfo(&pSql->cmd, 0)->name, 0); pRes->code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; - code = tscGetMetricMeta(pSql, pSql->cmd.name); + code = tscGetMetricMeta(pSql); pRes->code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; @@ -508,14 +504,17 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) { code = tsParseSql(pSql, pObj->acctId, pObj->db, false); if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; } + } else { // stream computing - code = tscGetMeterMeta(pSql, pSql->cmd.name); + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + code = tscGetMeterMeta(pSql, pMeterMetaInfo->name, 0); pRes->code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; - if (code == TSDB_CODE_SUCCESS && UTIL_METER_IS_METRIC(pCmd)) { - code = tscGetMetricMeta(pSql, pSql->cmd.name); + pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + if (code == TSDB_CODE_SUCCESS && UTIL_METER_IS_METRIC(pMeterMetaInfo)) { + code = tscGetMetricMeta(pSql); pRes->code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; diff --git a/src/client/src/tscCache.c b/src/client/src/tscCache.c index b58df68e48..866b6e7dbc 100644 --- a/src/client/src/tscCache.c +++ b/src/client/src/tscCache.c @@ -102,7 +102,12 @@ void *taosAddConnIntoCache(void *handle, void *data, uint32_t ip, short port, ch pObj = (SConnCache *)handle; if (pObj == NULL || pObj->maxSessions == 0) return NULL; + +#ifdef CLUSTER + if (data == NULL || ip == 0) { +#else if (data == NULL) { +#endif tscTrace("data:%p ip:%p:%d not valid, not added in cache", data, ip, port); return NULL; } diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 8e7ee8447a..cdc220441a 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -31,51 +32,192 @@ #include "thistogram.h" #include "tinterpolation.h" #include "tlog.h" +#include "tscJoinProcess.h" #include "tscSyntaxtreefunction.h" +#include "tscompression.h" #include "tsqlfunction.h" +#include "ttime.h" #include "ttypes.h" #include "tutil.h" +#define GET_INPUT_CHAR(x) (((char *)((x)->aInputElemBuf)) + ((x)->startOffset) * ((x)->inputBytes)) +#define GET_INPUT_CHAR_INDEX(x, y) (GET_INPUT_CHAR(x) + (y) * (x)->inputBytes) + +#define GET_TRUE_DATA_TYPE() \ + int32_t type = 0; \ + if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { \ + type = pCtx->outputType; \ + assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); \ + } else { \ + type = pCtx->inputType; \ + } + +#define SET_VAL(ctx, numOfElem, res) \ + do { \ + if ((numOfElem) <= 0) { \ + break; \ + } \ + GET_RES_INFO(ctx)->numOfRes = (res); \ + } while (0); + +#define INC_INIT_VAL(ctx, res) (GET_RES_INFO(ctx)->numOfRes += (res)); + +#define DO_UPDATE_TAG_COLUMNS(ctx, ts) \ + do { \ + for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \ + SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i]; \ + if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { \ + __ctx->tag = (tVariant){.i64Key = (ts), .nType = TSDB_DATA_TYPE_BIGINT}; \ + } \ + aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ + } \ + } while (0); + +void noop(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} + typedef struct tValuePair { tVariant v; int64_t timestamp; + char * pTags; // the corresponding tags of each record in the final result } tValuePair; -typedef struct SSpreadRuntime { - double start; - double end; - char valid; -} SSpreadRuntime; +typedef struct SSpreadInfo { + double min; + double max; + int8_t hasResult; +} SSpreadInfo; -void getResultInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, - int16_t *bytes) { +typedef struct SSumInfo { + union { + int64_t isum; + double dsum; + }; + int8_t hasResult; +} SSumInfo; + +// the attribute of hasResult is not needed since the num attribute would server as this purpose +typedef struct SAvgInfo { + double sum; + int64_t num; // num servers as the hasResult attribute in other struct +} SAvgInfo; + +typedef struct SStddevInfo { + double avg; + int64_t num; + double res; + int8_t stage; +} SStddevInfo; + +typedef struct SFirstLastInfo { + int8_t hasResult; + TSKEY ts; +} SFirstLastInfo; + +typedef struct SFirstLastInfo SLastrowInfo; +typedef struct SPercentileInfo { tMemBucket *pMemBucket; } SPercentileInfo; + +typedef struct STopBotInfo { + int32_t num; + tValuePair **res; +} STopBotInfo; + +// leastsquares do not apply to super table +typedef struct SLeastsquareInfo { + double mat[2][3]; + double startVal; + int64_t num; +} SLeastsquareInfo; + +typedef struct SAPercentileInfo { SHistogramInfo *pHisto; } SAPercentileInfo; + +typedef struct STSCompInfo { STSBuf *pTSBuf; } STSCompInfo; + +int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, + int16_t *bytes, int16_t *intermediateResBytes, int16_t extLength, bool isSuperTable) { if (!isValidDataType(dataType, dataBytes)) { pError("Illegal data type %d or data type length %d", dataType, dataBytes); - return; + return TSDB_CODE_INVALID_SQL; } - if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX || functionId == TSDB_FUNC_FIRST || - functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || - functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || - functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG || - functionId == TSDB_FUNC_INTERP || functionId == TSDB_FUNC_LAST_ROW) { + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG_DUMMY || + functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAGPRJ || + functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; - return; + *intermediateResBytes = *bytes + sizeof(SResultInfo); + return TSDB_CODE_SUCCESS; } if (functionId == TSDB_FUNC_COUNT) { *type = TSDB_DATA_TYPE_BIGINT; *bytes = sizeof(int64_t); - return; + *intermediateResBytes = *bytes; + return TSDB_CODE_SUCCESS; } - if (functionId == TSDB_FUNC_AVG || functionId == TSDB_FUNC_PERCT || functionId == TSDB_FUNC_APERCT || - functionId == TSDB_FUNC_STDDEV || functionId == TSDB_FUNC_ARITHM || functionId == TSDB_FUNC_SPREAD || - functionId == TSDB_FUNC_WAVG) { + if (functionId == TSDB_FUNC_ARITHM) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); - return; + *intermediateResBytes = *bytes; + return TSDB_CODE_SUCCESS; + } + + if (functionId == TSDB_FUNC_TS_COMP) { + *type = TSDB_DATA_TYPE_BINARY; + *bytes = sizeof(int32_t); // this results is compressed ts data + *intermediateResBytes = POINTER_BYTES; + return TSDB_CODE_SUCCESS; + } + + if (isSuperTable) { + if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) { + *type = TSDB_DATA_TYPE_BINARY; + *bytes = dataBytes + DATA_SET_FLAG_SIZE; + *intermediateResBytes = *bytes; + + return TSDB_CODE_SUCCESS; + } else if (functionId == TSDB_FUNC_SUM) { + *type = TSDB_DATA_TYPE_BINARY; + *bytes = sizeof(SSumInfo); + *intermediateResBytes = *bytes; + + return TSDB_CODE_SUCCESS; + } else if (functionId == TSDB_FUNC_AVG) { + *type = TSDB_DATA_TYPE_BINARY; + *bytes = sizeof(SAvgInfo); + *intermediateResBytes = *bytes; + + return TSDB_CODE_SUCCESS; + } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + *type = TSDB_DATA_TYPE_BINARY; + *bytes = sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param; + *intermediateResBytes = *bytes; + + return TSDB_CODE_SUCCESS; + } else if (functionId == TSDB_FUNC_SPREAD) { + *type = TSDB_DATA_TYPE_BINARY; + *bytes = sizeof(SSpreadInfo); + *intermediateResBytes = *bytes; + + return TSDB_CODE_SUCCESS; + } else if (functionId == TSDB_FUNC_APERCT) { + *type = TSDB_DATA_TYPE_BINARY; + *bytes = sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1) + sizeof(SHistogramInfo) + sizeof(SAPercentileInfo); + *intermediateResBytes = *bytes; + + return TSDB_CODE_SUCCESS; + } else if (functionId == TSDB_FUNC_LAST_ROW) { + *type = TSDB_DATA_TYPE_BINARY; + *bytes = sizeof(SLastrowInfo) + dataBytes; + *intermediateResBytes = *bytes; + + return TSDB_CODE_SUCCESS; + } else if (functionId == TSDB_FUNC_TWA) { + *type = TSDB_DATA_TYPE_DOUBLE; + *bytes = sizeof(STwaInfo); + *intermediateResBytes = sizeof(STwaInfo); + return TSDB_CODE_SUCCESS; + } } if (functionId == TSDB_FUNC_SUM) { @@ -86,162 +228,197 @@ void getResultInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int3 } *bytes = sizeof(int64_t); - return; + *intermediateResBytes = sizeof(SSumInfo); + return TSDB_CODE_SUCCESS; + } else if (functionId == TSDB_FUNC_APERCT) { + *type = TSDB_DATA_TYPE_DOUBLE; + *bytes = sizeof(double); + *intermediateResBytes = + sizeof(SAPercentileInfo) + sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1); + return TSDB_CODE_SUCCESS; + } else if (functionId == TSDB_FUNC_TWA) { + *type = TSDB_DATA_TYPE_DOUBLE; + *bytes = sizeof(double); + *intermediateResBytes = sizeof(STwaInfo); + return TSDB_CODE_SUCCESS; } - if (functionId == TSDB_FUNC_LEASTSQR) { + if (functionId == TSDB_FUNC_AVG) { + *type = TSDB_DATA_TYPE_DOUBLE; + *bytes = sizeof(double); + *intermediateResBytes = sizeof(SAvgInfo); + } else if (functionId == TSDB_FUNC_STDDEV) { + *type = TSDB_DATA_TYPE_DOUBLE; + *bytes = sizeof(double); + *intermediateResBytes = sizeof(SStddevInfo); + } else if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) { + *type = (int16_t)dataType; + *bytes = (int16_t)dataBytes; + *intermediateResBytes = dataBytes + DATA_SET_FLAG_SIZE; + } else if (functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_LAST) { + *type = (int16_t)dataType; + *bytes = (int16_t)dataBytes; + *intermediateResBytes = dataBytes + sizeof(SResultInfo); + } else if (functionId == TSDB_FUNC_SPREAD) { + *type = (int16_t)TSDB_DATA_TYPE_DOUBLE; + *bytes = sizeof(double); + *intermediateResBytes = sizeof(SSpreadInfo); + } else if (functionId == TSDB_FUNC_PERCT) { + *type = (int16_t)TSDB_DATA_TYPE_DOUBLE; + *bytes = (int16_t)sizeof(double); + *intermediateResBytes = POINTER_BYTES; + } else if (functionId == TSDB_FUNC_LEASTSQR) { *type = TSDB_DATA_TYPE_BINARY; *bytes = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE; // string + *intermediateResBytes = *bytes + sizeof(SResultInfo); } else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { *type = TSDB_DATA_TYPE_BINARY; - *bytes = dataBytes + DATA_SET_FLAG_SIZE + TSDB_KEYSIZE; - } else if (functionId == TSDB_FUNC_SPREAD_DST) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(SSpreadRuntime); - } else if (functionId == TSDB_FUNC_WAVG_DST) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(SWavgRuntime); - } else if (functionId == TSDB_FUNC_MIN_DST || functionId == TSDB_FUNC_MAX_DST) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = dataBytes + DATA_SET_FLAG_SIZE; - } else if (functionId == TSDB_FUNC_SUM_DST) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(SSumRuntime); - } else if (functionId == TSDB_FUNC_AVG_DST) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(SAvgRuntime); - } else if (functionId == TSDB_FUNC_TOP_DST || functionId == TSDB_FUNC_BOTTOM_DST) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(int64_t) + sizeof(tValuePair) * param; - } else if (functionId == TSDB_FUNC_APERCT_DST) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1) + sizeof(SHistogramInfo); - } else if (functionId == TSDB_FUNC_LAST_ROW_DST) { - *type = TSDB_DATA_TYPE_BINARY; - *bytes = dataBytes + DATA_SET_FLAG_SIZE + TSDB_KEYSIZE; + *bytes = dataBytes + sizeof(SFirstLastInfo); + *intermediateResBytes = *bytes; + } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + *type = (int16_t)dataType; + *bytes = (int16_t)dataBytes; + + size_t size = sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param; + + // the output column may be larger than sizeof(STopBotInfo) + *intermediateResBytes = size; + } else if (functionId == TSDB_FUNC_LAST_ROW) { + *type = (int16_t)dataType; + *bytes = (int16_t)dataBytes; + *intermediateResBytes = dataBytes + sizeof(SLastrowInfo); + } else { + return TSDB_CODE_INVALID_SQL; } + + return TSDB_CODE_SUCCESS; } -/* - * whether has put the first result into the output buffer - * decided by if there are timestamp of value and the seperator ',' +/** + * the numOfRes should be kept, since it may be used later + * and allow the ResultInfo to be re initialized */ +void resetResultInfo(SResultInfo *pResInfo) { pResInfo->initialized = false; } -#define IS_DATA_NOT_ASSIGNED(ctx) (*(char *)((ctx)->aOutputBuf + TSDB_KEYSIZE) != DATA_SET_FLAG) -#define SET_DATA_ASSIGNED(ctx) (*(char *)((ctx)->aOutputBuf + TSDB_KEYSIZE) = DATA_SET_FLAG) - -#define SET_VAL(ctx, numOfElem, numOfRes) \ - do { \ - if ((numOfElem) <= 0) { \ - break; \ - } \ - (ctx)->numOfIteratedElems += (numOfElem); \ - (ctx)->numOfOutputElems = (numOfRes); \ - } while (0); +void initResultInfo(SResultInfo *pResInfo) { + pResInfo->initialized = true; // the this struct has been initialized flag -#define INC_INIT_VAL(ctx, numOfElem, numOfRes) \ - do { \ - (ctx)->numOfIteratedElems += (numOfElem); \ - (ctx)->numOfOutputElems += (numOfRes); \ - } while (0); + pResInfo->complete = false; + pResInfo->hasResult = false; + pResInfo->numOfRes = 0; -void noop(SQLFunctionCtx *pCtx) { UNUSED(pCtx); /* suppress warning*/ } -bool no_next_step(SQLFunctionCtx *pCtx) { - UNUSED(pCtx); - return false; + memset(pResInfo->interResultBuf, 0, (size_t)pResInfo->bufLen); } -#define INIT_VAL(ctx) \ - do { \ - (ctx)->currentStage = 0; \ - (ctx)->numOfOutputElems = 0; \ - (ctx)->numOfIteratedElems = 0; \ - } while (0); +void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable) { + assert(pResInfo->interResultBuf == NULL); -#define GET_INPUT_CHAR(x) (((char *)((x)->aInputElemBuf)) + ((x)->startOffset) * ((x)->inputBytes)) -#define GET_INPUT_CHAR_INDEX(x, y) (GET_INPUT_CHAR(x) + (y) * (x)->inputBytes) + pResInfo->bufLen = size; + pResInfo->superTableQ = superTable; + + pResInfo->interResultBuf = calloc(1, (size_t)size); +} + +// set the query flag to denote that query is completed +static void no_next_step(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->complete = true; +} -#define SET_HAS_DATA_FLAG(x) ((x) = DATA_SET_FLAG) -#define HAS_DATA_FLAG(x) ((x) == DATA_SET_FLAG) +static bool function_setup(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->initialized) { + return false; + } -#define tPow(x) ((x) * (x)) + memset(pCtx->aOutputBuf, 0, (size_t)pCtx->outputBytes); -void function_setup(SQLFunctionCtx *pCtx) { - memset(pCtx->aOutputBuf, 0, pCtx->outputBytes); - pCtx->intermediateBuf[0].i64Key = 0; - INIT_VAL(pCtx); + initResultInfo(pResInfo); + return true; } /** - * in handling the stable query, function_finalize is called after the secondary + * in handling the stable query, function_finalizer is called after the secondary * merge being completed, during the first merge procedure, which is executed at the * vnode side, the finalize will never be called. - * todo: add more information for debug in SQLFunctionCtx + * * @param pCtx */ -void function_finalize(SQLFunctionCtx *pCtx) { - if (pCtx->numOfIteratedElems == 0) { +static void function_finalizer(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + + if (pResInfo->hasResult != DATA_SET_FLAG) { pTrace("no result generated, result is set to NULL"); setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); } + + resetResultInfo(GET_RES_INFO(pCtx)); } -static bool count_function(SQLFunctionCtx *pCtx) { +/* + * count function does need the finalize, if data is missing, the default value, which is 0, is used + * count function does not use the pCtx->interResBuf to keep the intermediate buffer + */ +static void count_function(SQLFunctionCtx *pCtx) { int32_t numOfElem = 0; - /* - * 1. column data missing (schema modified) causes pCtx->hasNullValue == true. pCtx->preAggVals.isSet == true; - * 2. for general non-primary key columns, pCtx->hasNullValue may be true or false, pCtx->preAggVals.isSet == true; - * 3. for primary key column, pCtx->hasNullValue always be false, pCtx->preAggVals.isSet == false; - */ - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus)) { // Pre-aggregation + if (IS_DATA_BLOCK_LOADED(pCtx->blockStatus)) { + /* + * In following cases, the data block is loaded: + * 1. A first/last file block for a query + * 2. Required to handle other queries, such as apercentile/twa/stddev etc. + * 3. A cache block + */ + if (pCtx->hasNull) { + for (int32_t i = 0; i < pCtx->size; ++i) { + char *val = GET_INPUT_CHAR_INDEX(pCtx, i); + if (isNull(val, pCtx->inputType)) { + continue; + } + + numOfElem += 1; + } + } else { + numOfElem = pCtx->size; + } + } else { + /* + * 1. column data missing (schema modified) causes pCtx->hasNull == true. pCtx->preAggVals.isSet == true; + * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->preAggVals.isSet == true; + * 3. for primary key column, pCtx->hasNull always be false, pCtx->preAggVals.isSet == false; + */ if (pCtx->preAggVals.isSet) { - numOfElem = pCtx->size - pCtx->preAggVals.numOfNullPoints; + numOfElem = pCtx->size - pCtx->preAggVals.numOfNull; } else { - assert(pCtx->hasNullValue == false); + assert(pCtx->hasNull == false); numOfElem = pCtx->size; } - - goto _count_over; } - /* - * In following cases, the data block is loaded: - * 1. it is a first/last file block for a query - * 2. actual block data is loaded in case of handling other queries, such as apercentile/wavg/stddev etc. - * 3. it is a cache block - */ - if (pCtx->hasNullValue) { - for (int32_t i = 0; i < pCtx->size; ++i) { - char *val = GET_INPUT_CHAR_INDEX(pCtx, i); - if (isNull(val, pCtx->inputType)) { - continue; - } - - numOfElem += 1; - } - } else { - numOfElem = pCtx->size; + if (numOfElem > 0) { + GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } -_count_over: *((int64_t *)pCtx->aOutputBuf) += numOfElem; SET_VAL(pCtx, numOfElem, 1); - return true; } -static bool count_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } SET_VAL(pCtx, 1, 1); + *((int64_t *)pCtx->aOutputBuf) += 1; - return true; + + // do not need it actually + SResultInfo *pInfo = GET_RES_INFO(pCtx); + pInfo->hasResult = DATA_SET_FLAG; } -static void count_dist_merge(SQLFunctionCtx *pCtx) { +static void count_func_merge(SQLFunctionCtx *pCtx) { int64_t *pData = (int64_t *)GET_INPUT_CHAR(pCtx); for (int32_t i = 0; i < pCtx->size; ++i) { *((int64_t *)pCtx->aOutputBuf) += pData[i]; @@ -252,7 +429,7 @@ static void count_dist_merge(SQLFunctionCtx *pCtx) { /** * 1. If the column value for filter exists, we need to load the SFields, which serves - * as the pre-filter to decide if the actual data block is required or not. + * as the pre-filter to decide if the actual data block is required or not. * 2. If it queries on the non-primary timestamp column, SFields is also required to get the not-null value. * * @param colId @@ -271,192 +448,152 @@ int32_t no_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId return BLK_DATA_NO_NEEDED; } -#define LIST_ADD(x, n, p) \ - for (int32_t i = 0; i < (n); ++i) { \ - (x) += (p)[i]; \ - }; - -#define TYPED_LIST_ADD(x, n, s, t) \ - do { \ - t *p = (t *)s; \ - LIST_ADD(x, n, p); \ - } while (0); - -#define LIST_ADD_N(x, n, p, t, numOfElem, tsdbType) \ - { \ - t *d = (t *)(p); \ - for (int32_t i = 0; i < (n); ++i) { \ - if (isNull((char *)&(d)[i], tsdbType)) { \ - continue; \ - }; \ - (x) += (d)[i]; \ - numOfElem++; \ - } \ +#define LIST_ADD_N(x, ctx, p, t, numOfElem, tsdbType) \ + { \ + t *d = (t *)(p); \ + for (int32_t i = 0; i < (ctx)->size; ++i) { \ + if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ + continue; \ + }; \ + (x) += (d)[i]; \ + (numOfElem)++; \ + } \ }; -#define LOOPCHECK(v, d, n, sign) \ - for (int32_t i = 0; i < (n); ++i) { \ - (v) = (((v) < (d)[i]) ^ (sign)) ? (d)[i] : (v); \ - } +#define UPDATE_DATA(ctx, left, right, num, sign) \ + do { \ + if (((left) < (right)) ^ (sign)) { \ + (left) = right; \ + DO_UPDATE_TAG_COLUMNS(ctx, 0); \ + (num) += 1; \ + } \ + } while (0) -#define LOOPCHECK_N(val, list, num, tsdbType, sign, notNullElem) \ - for (int32_t i = 0; i < (num); ++i) { \ - if (isNull((char *)&(list)[i], tsdbType)) { \ - continue; \ - } \ - (val) = (((val) < (list)[i]) ^ (sign)) ? (list)[i] : (val); \ - notNullElem += 1; \ +#define LOOPCHECK_N(val, list, ctx, tsdbType, sign, num) \ + for (int32_t i = 0; i < ((ctx)->size); ++i) { \ + if ((ctx)->hasNull && isNull((char *)&(list)[i], tsdbType)) { \ + continue; \ + } \ + UPDATE_DATA(ctx, val, (list)[i], num, sign); \ } -#define TYPED_LOOPCHECK(t, v, d, n, sign) \ - do { \ - t *_d = (t *)d; \ - t *v1 = (t *)v; \ - LOOPCHECK(*v1, _d, n, sign); \ - } while (0) - -#define TYPED_LOOPCHECK_N(type, data, list, num, tsdbType, sign, notNullElems) \ +#define TYPED_LOOPCHECK_N(type, data, list, ctx, tsdbType, sign, notNullElems) \ do { \ type *_data = (type *)data; \ type *_list = (type *)list; \ - LOOPCHECK_N(*_data, _list, num, tsdbType, sign, notNullElems); \ + LOOPCHECK_N(*_data, _list, ctx, tsdbType, sign, notNullElems); \ } while (0) -static bool sum_function(SQLFunctionCtx *pCtx) { +static void do_sum(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; + // Only the pre-computing information loaded and actual data does not loaded if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) && pCtx->preAggVals.isSet) { - // it's the whole block to be calculated, so the assert must be correct - assert(pCtx->size >= pCtx->preAggVals.numOfNullPoints); - notNullElems = (pCtx->size - pCtx->preAggVals.numOfNullPoints); - - if (notNullElems > 0) { - if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - int64_t *retVal = pCtx->aOutputBuf; - *retVal += pCtx->preAggVals.sum; - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = pCtx->aOutputBuf; - *retVal += *(double *)&(pCtx->preAggVals.sum); - } - } - goto _sum_over; - } - - void *pData = GET_INPUT_CHAR(pCtx); - - if (pCtx->hasNullValue) { - notNullElems = 0; + notNullElems = pCtx->size - pCtx->preAggVals.numOfNull; + assert(pCtx->size >= pCtx->preAggVals.numOfNull); if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { int64_t *retVal = pCtx->aOutputBuf; - - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_ADD_N(*retVal, pCtx->size, pData, int8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_ADD_N(*retVal, pCtx->size, pData, int16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_ADD_N(*retVal, pCtx->size, pData, int32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - LIST_ADD_N(*retVal, pCtx->size, pData, int64_t, notNullElems, pCtx->inputType); - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *retVal = pCtx->aOutputBuf; - LIST_ADD_N(*retVal, pCtx->size, pData, double, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { + *retVal += pCtx->preAggVals.sum; + } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { double *retVal = pCtx->aOutputBuf; - LIST_ADD_N(*retVal, pCtx->size, pData, float, notNullElems, pCtx->inputType); + *retVal += GET_DOUBLE_VAL(&(pCtx->preAggVals.sum)); } - } else { - notNullElems = pCtx->size; + } else { // computing based on the true data block + void *pData = GET_INPUT_CHAR(pCtx); + notNullElems = 0; if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { int64_t *retVal = pCtx->aOutputBuf; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - TYPED_LIST_ADD(*retVal, pCtx->size, pData, int8_t); + LIST_ADD_N(*retVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - TYPED_LIST_ADD(*retVal, pCtx->size, pData, int16_t); + LIST_ADD_N(*retVal, pCtx, pData, int16_t, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - TYPED_LIST_ADD(*retVal, pCtx->size, pData, int32_t); + LIST_ADD_N(*retVal, pCtx, pData, int32_t, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - TYPED_LIST_ADD(*retVal, pCtx->size, pData, int64_t); + LIST_ADD_N(*retVal, pCtx, pData, int64_t, notNullElems, pCtx->inputType); } } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { double *retVal = pCtx->aOutputBuf; - TYPED_LIST_ADD(*retVal, pCtx->size, pData, double); + LIST_ADD_N(*retVal, pCtx, pData, double, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { double *retVal = pCtx->aOutputBuf; - TYPED_LIST_ADD(*retVal, pCtx->size, pData, float); + LIST_ADD_N(*retVal, pCtx, pData, float, notNullElems, pCtx->inputType); } } -_sum_over: // data in the check operation are all null, not output SET_VAL(pCtx, notNullElems, 1); - return true; + + if (notNullElems > 0) { + GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; + } } -static bool sum_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } SET_VAL(pCtx, 1, 1); int64_t *res = pCtx->aOutputBuf; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - *res += *(int8_t *)pData; + *res += GET_INT8_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - *res += *(int16_t *)pData; + *res += GET_INT16_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - *res += *(int32_t *)pData; + *res += GET_INT32_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - *res += *(int64_t *)pData; + *res += GET_INT64_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { double *retVal = pCtx->aOutputBuf; - *retVal += *(double *)pData; + *retVal += GET_DOUBLE_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { double *retVal = pCtx->aOutputBuf; - *retVal += *(float *)pData; + *retVal += GET_FLOAT_VAL(pData); } - return true; + GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } -static bool sum_dist_intern_function(SQLFunctionCtx *pCtx) { - sum_function(pCtx); +static void sum_function(SQLFunctionCtx *pCtx) { + do_sum(pCtx); // keep the result data in output buffer, not in the intermediate buffer - if (pCtx->numOfIteratedElems > 0) { - char *pOutputBuf = pCtx->aOutputBuf; - *(pOutputBuf + sizeof(double)) = DATA_SET_FLAG; + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { + // set the flag for super table query + SSumInfo *pSum = (SSumInfo *)pCtx->aOutputBuf; + pSum->hasResult = DATA_SET_FLAG; } - - return true; } -static bool sum_dist_intern_function_f(SQLFunctionCtx *pCtx, int32_t index) { - sum_function_f(pCtx, index); +static void sum_function_f(SQLFunctionCtx *pCtx, int32_t index) { + do_sum_f(pCtx, index); - /* keep the result data in output buffer, not in the intermediate buffer */ - if (pCtx->numOfIteratedElems) { - char *pOutputBuf = pCtx->aOutputBuf; - *(pOutputBuf + sizeof(double)) = DATA_SET_FLAG; + // keep the result data in output buffer, not in the intermediate buffer + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { + SSumInfo *pSum = (SSumInfo *)pCtx->aOutputBuf; + pSum->hasResult = DATA_SET_FLAG; } - - return true; } -static int32_t do_sum_merge_impl(const SQLFunctionCtx *pCtx) { +static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - int32_t type = (pCtx->outputType != TSDB_DATA_TYPE_BINARY) ? pCtx->outputType : pCtx->inputType; - char * input = GET_INPUT_CHAR(pCtx); + GET_TRUE_DATA_TYPE(); + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pResInfo->superTableQ); - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SSumRuntime *pInput = (SSumRuntime *)input; - if (pInput->valFlag != DATA_SET_FLAG) { + for (int32_t i = 0; i < pCtx->size; ++i) { + char * input = GET_INPUT_CHAR_INDEX(pCtx, i); + SSumInfo *pInput = (SSumInfo *)input; + if (pInput->hasResult != DATA_SET_FLAG) { continue; } @@ -467,37 +604,39 @@ static int32_t do_sum_merge_impl(const SQLFunctionCtx *pCtx) { case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_BIGINT: { - *(int64_t *)pCtx->aOutputBuf += pInput->iOutput; + *(int64_t *)pCtx->aOutputBuf += pInput->isum; break; }; case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: { - *(double *)pCtx->aOutputBuf += pInput->dOutput; + *(double *)pCtx->aOutputBuf += pInput->dsum; } } } + return notNullElems; } -static void sum_dist_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = do_sum_merge_impl(pCtx); +static void sum_func_merge(SQLFunctionCtx *pCtx) { + int32_t notNullElems = sum_merge_impl(pCtx); SET_VAL(pCtx, notNullElems, 1); - SSumRuntime *pSumRuntime = (SSumRuntime *)pCtx->aOutputBuf; + SSumInfo *pSumInfo = (SSumInfo *)pCtx->aOutputBuf; if (notNullElems > 0) { - pCtx->numOfIteratedElems += notNullElems; - pSumRuntime->valFlag = DATA_SET_FLAG; + // pCtx->numOfIteratedElems += notNullElems; + pSumInfo->hasResult = DATA_SET_FLAG; } } -static void sum_dist_second_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = do_sum_merge_impl(pCtx); +static void sum_func_second_merge(SQLFunctionCtx *pCtx) { + int32_t notNullElems = sum_merge_impl(pCtx); SET_VAL(pCtx, notNullElems, 1); - /* NOTE: no flag value exists for secondary merge */ + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + if (notNullElems > 0) { - pCtx->numOfIteratedElems += notNullElems; + pResInfo->hasResult = DATA_SET_FLAG; } } @@ -515,8 +654,8 @@ static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, return BLK_DATA_NO_NEEDED; } - /* no result for first query, data block is required */ - if (pCtx->numOfOutputElems <= 0) { + // no result for first query, data block is required + if (GET_RES_INFO(pCtx)->numOfRes <= 0) { return BLK_DATA_ALL_NEEDED; } else { return BLK_DATA_NO_NEEDED; @@ -528,7 +667,7 @@ static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, return BLK_DATA_NO_NEEDED; } - if (pCtx->numOfOutputElems <= 0) { + if (GET_RES_INFO(pCtx)->numOfRes <= 0) { return BLK_DATA_ALL_NEEDED; } else { return BLK_DATA_NO_NEEDED; @@ -541,12 +680,11 @@ static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY return BLK_DATA_NO_NEEDED; } - if (IS_DATA_NOT_ASSIGNED(pCtx)) { + SFirstLastInfo *pInfo = (pCtx->aOutputBuf + pCtx->inputBytes); + if (pInfo->hasResult != DATA_SET_FLAG) { return BLK_DATA_ALL_NEEDED; - } else { - // data in current block is not earlier than current result - TSKEY ts = *(TSKEY *)pCtx->aOutputBuf; - return (ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; + } else { // data in current block is not earlier than current result + return (pInfo->ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; } } @@ -556,446 +694,445 @@ static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY return BLK_DATA_NO_NEEDED; } - if (IS_DATA_NOT_ASSIGNED(pCtx)) { + SFirstLastInfo *pInfo = (pCtx->aOutputBuf + pCtx->inputBytes); + if (pInfo->hasResult != DATA_SET_FLAG) { return BLK_DATA_ALL_NEEDED; } else { - TSKEY ts = *(TSKEY *)pCtx->aOutputBuf; - return (ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; - } -} - -/* - * the average value is calculated in finalize routine, since current routine does not know the exact number of points - */ -static void avg_finalizer(SQLFunctionCtx *pCtx) { - // pCtx->numOfIteratedElems is the number of not null elements in current - // query range - if (pCtx->numOfIteratedElems == 0) { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); - return; // empty table - } - - if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - int64_t *retVal = pCtx->aOutputBuf; - *(double *)pCtx->aOutputBuf = (*retVal) / (double)pCtx->numOfIteratedElems; - } else { - double *retVal = pCtx->aOutputBuf; - *retVal = *retVal / (double)pCtx->numOfIteratedElems; - } - - /* cannot set the numOfIteratedElems again since it is set during previous iteration */ - pCtx->numOfOutputElems = 1; -} - -static void avg_dist_function_setup(SQLFunctionCtx *pCtx) { - pCtx->intermediateBuf[0].nType = TSDB_DATA_TYPE_DOUBLE; - memset(pCtx->aOutputBuf, 0, pCtx->outputBytes); - INIT_VAL(pCtx); -} - -static void avg_dist_merge(SQLFunctionCtx *pCtx) { - SAvgRuntime *pDest = (SAvgRuntime *)pCtx->aOutputBuf; - - char *input = GET_INPUT_CHAR(pCtx); - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SAvgRuntime *pInput = (SAvgRuntime *)input; - if (pInput->valFlag != DATA_SET_FLAG) { // current buffer is null - continue; - } - - pDest->sum += pInput->sum; - pDest->num += pInput->num; - pDest->valFlag = DATA_SET_FLAG; - } - - /* if the data set flag is not set, the result is null */ - pCtx->numOfIteratedElems = pDest->num; -} - -static void avg_dist_second_merge(SQLFunctionCtx *pCtx) { - double *sum = pCtx->aOutputBuf; - char * input = GET_INPUT_CHAR(pCtx); - - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SAvgRuntime *pInput = (SAvgRuntime *)input; - if (pInput->valFlag != DATA_SET_FLAG) { // current input is null - continue; - } - - *sum += pInput->sum; - pCtx->numOfIteratedElems += pInput->num; + return (pInfo->ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; } } +////////////////////////////////////////////////////////////////////////////////////////////// /* - * there is only tiny difference between avg_dist_intern_function and sum_function, - * the output type + * The intermediate result of average is kept in the interResultBuf. + * For super table query, once the avg_function/avg_function_f is finished, copy the intermediate + * result into output buffer. */ -static bool avg_dist_intern_function(SQLFunctionCtx *pCtx) { +static void avg_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - double *retVal = pCtx->aOutputBuf; + // NOTE: keep the intermediate result into the interResultBuf + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + + SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; + double * pVal = &pAvgInfo->sum; if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) && pCtx->preAggVals.isSet) { // Pre-aggregation - notNullElems = pCtx->size - pCtx->preAggVals.numOfNullPoints; + notNullElems = pCtx->size - pCtx->preAggVals.numOfNull; assert(notNullElems >= 0); if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - *retVal += pCtx->preAggVals.sum; + *pVal += pCtx->preAggVals.sum; } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - *retVal += *(double *)&(pCtx->preAggVals.sum); - } else { - return false; - } - - goto _sum_over; - } - - void *pData = GET_INPUT_CHAR(pCtx); - - if (pCtx->hasNullValue) { - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_ADD_N(*retVal, pCtx->size, pData, int8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_ADD_N(*retVal, pCtx->size, pData, int16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_ADD_N(*retVal, pCtx->size, pData, int32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - LIST_ADD_N(*retVal, pCtx->size, pData, int64_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - LIST_ADD_N(*retVal, pCtx->size, pData, double, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - LIST_ADD_N(*retVal, pCtx->size, pData, float, notNullElems, pCtx->inputType); + *pVal += GET_DOUBLE_VAL(&(pCtx->preAggVals.sum)); } } else { - notNullElems = pCtx->size; + void *pData = GET_INPUT_CHAR(pCtx); if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - TYPED_LIST_ADD(*retVal, pCtx->size, pData, int8_t); + LIST_ADD_N(*pVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - TYPED_LIST_ADD(*retVal, pCtx->size, pData, int16_t); + LIST_ADD_N(*pVal, pCtx, pData, int16_t, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - TYPED_LIST_ADD(*retVal, pCtx->size, pData, int32_t); + LIST_ADD_N(*pVal, pCtx, pData, int32_t, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - TYPED_LIST_ADD(*retVal, pCtx->size, pData, int64_t); + LIST_ADD_N(*pVal, pCtx, pData, int64_t, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - TYPED_LIST_ADD(*retVal, pCtx->size, pData, double); + LIST_ADD_N(*pVal, pCtx, pData, double, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - TYPED_LIST_ADD(*retVal, pCtx->size, pData, float); + LIST_ADD_N(*pVal, pCtx, pData, float, notNullElems, pCtx->inputType); } } -_sum_over: - if (notNullElems > 0) { - SET_VAL(pCtx, notNullElems, 1); - SAvgRuntime *pAvgRuntime = (SAvgRuntime *)pCtx->aOutputBuf; - pAvgRuntime->num += notNullElems; + if (!pCtx->hasNull) { + assert(notNullElems == pCtx->size); + } + + SET_VAL(pCtx, notNullElems, 1); + pAvgInfo->num += notNullElems; - // the delimiter of ',' is used to denote current buffer has output or not - pAvgRuntime->valFlag = DATA_SET_FLAG; + if (notNullElems > 0) { + pResInfo->hasResult = DATA_SET_FLAG; } - return true; + // keep the data into the final output buffer for super table query since this execution may be the last one + if (pResInfo->superTableQ) { + memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo)); + } } -static bool avg_dist_intern_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } SET_VAL(pCtx, 1, 1); - SAvgRuntime *pDest = (SAvgRuntime *)pCtx->aOutputBuf; + + // NOTE: keep the intermediate result into the interResultBuf + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + + SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - pDest->sum += *(int8_t *)pData; + pAvgInfo->sum += GET_INT8_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - pDest->sum += *(int16_t *)pData; + pAvgInfo->sum += GET_INT16_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - pDest->sum += *(int32_t *)pData; + pAvgInfo->sum += GET_INT32_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - pDest->sum += *(int64_t *)pData; + pAvgInfo->sum += GET_INT64_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - pDest->sum += *(double *)pData; + pAvgInfo->sum += GET_DOUBLE_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - pDest->sum += *(float *)pData; + pAvgInfo->sum += GET_FLOAT_VAL(pData); } // restore sum and count of elements - pDest->num += 1; - pDest->valFlag = DATA_SET_FLAG; - return true; -} - -///////////////////////////////////////////////////////////////////////////////////////////// + pAvgInfo->num += 1; -static bool minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) { - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) && pCtx->preAggVals.isSet) { // pre-agg - /* data in current data block are qualified to the query */ - *notNullElems = pCtx->size - pCtx->preAggVals.numOfNullPoints; - assert(*notNullElems >= 0); + // set has result flag + pResInfo->hasResult = DATA_SET_FLAG; - void *tval = (void *)(isMin ? &pCtx->preAggVals.min : &pCtx->preAggVals.max); - if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - int64_t val = *(int64_t *)tval; - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - int8_t *data = (int8_t *)pOutput; - *data = (*data < val) ^ isMin ? val : *data; - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - int16_t *data = (int16_t *)pOutput; - *data = (*data < val) ^ isMin ? val : *data; - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *data = (int32_t *)pOutput; - *data = (*data < val) ^ isMin ? val : *data; -#if defined(_DEBUG_VIEW) - pTrace("max value updated according to pre-cal:%d", *data); -#endif + // keep the data into the final output buffer for super table query since this execution may be the last one + if (pResInfo->superTableQ) { + memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo)); + } +} - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - int64_t *data = (int64_t *)pOutput; - *data = (*data < val) ^ isMin ? val : *data; - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *data = (double *)pOutput; - double val = *(double *)tval; +static void avg_func_merge(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pResInfo->superTableQ); - *data = (*data < val) ^ isMin ? val : *data; - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - float *data = (float *)pOutput; - double val = *(double *)tval; + SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; + char * input = GET_INPUT_CHAR(pCtx); - *data = (*data < val) ^ isMin ? val : *data; - } else { - return false; + for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { + SAvgInfo *pInput = (SAvgInfo *)input; + if (pInput->num == 0) { // current buffer is null + continue; } - return true; + pAvgInfo->sum += pInput->sum; + pAvgInfo->num += pInput->num; } - void *p = GET_INPUT_CHAR(pCtx); - if (pCtx->hasNullValue) { - *notNullElems = 0; - if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - TYPED_LOOPCHECK_N(int8_t, pOutput, p, pCtx->size, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - TYPED_LOOPCHECK_N(int16_t, pOutput, p, pCtx->size, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *pData = p; - int32_t *retVal = pOutput; + // if the data set hasResult is not set, the result is null + if (pAvgInfo->num > 0) { + pResInfo->hasResult = DATA_SET_FLAG; + memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo)); + } +} - for (int32_t i = 0; i < pCtx->size; ++i) { - if (isNull(&pData[i], pCtx->inputType)) { - continue; - } +static void avg_func_second_merge(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); - *retVal = ((*retVal < pData[i]) ^ isMin) ? pData[i] : *retVal; - *notNullElems += 1; - } -#if defined(_DEBUG_VIEW) - pTrace("max value updated:%d", *retVal); -#endif - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - TYPED_LOOPCHECK_N(int64_t, pOutput, p, pCtx->size, pCtx->inputType, isMin, *notNullElems); - } + double *sum = pCtx->aOutputBuf; + char * input = GET_INPUT_CHAR(pCtx); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - TYPED_LOOPCHECK_N(double, pOutput, p, pCtx->size, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - TYPED_LOOPCHECK_N(float, pOutput, p, pCtx->size, pCtx->inputType, isMin, *notNullElems); + for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { + SAvgInfo *pInput = (SAvgInfo *)input; + if (pInput->num == 0) { // current input is null + continue; } - } else { - *notNullElems = pCtx->size; + + *sum += pInput->sum; + + // keep the number of data into the temp buffer + *(int64_t *)pResInfo->interResultBuf += pInput->num; + } +} + +/* + * the average value is calculated in finalize routine, since current routine does not know the exact number of points + */ +static void avg_finalizer(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + + if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); + + if (GET_INT64_VAL(pResInfo->interResultBuf) <= 0) { + setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + return; // empty table + } + + *(double *)pCtx->aOutputBuf = (*(double *)pCtx->aOutputBuf) / *(int64_t *)pResInfo->interResultBuf; + } else { // this is the secondary merge, only in the secondary merge, the input type is TSDB_DATA_TYPE_BINARY + assert(pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE); + + SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; + + if (pAvgInfo->num == 0) { // all data are NULL or empty table + setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + return; + } + + *(double *)pCtx->aOutputBuf = pAvgInfo->sum / pAvgInfo->num; + } + + // cannot set the numOfIteratedElems again since it is set during previous iteration + GET_RES_INFO(pCtx)->numOfRes = 1; +} + +///////////////////////////////////////////////////////////////////////////////////////////// + +static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) { + if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) && pCtx->preAggVals.isSet) { + // data in current data block are qualified to the query + *notNullElems = pCtx->size - pCtx->preAggVals.numOfNull; + assert(*notNullElems >= 0); + + void * tval = NULL; + int16_t index = 0; + + if (isMin) { + tval = &pCtx->preAggVals.min; + index = pCtx->preAggVals.minIndex; + } else { + tval = &pCtx->preAggVals.max; + index = pCtx->preAggVals.maxIndex; + } + if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { + int64_t val = GET_INT64_VAL(tval); if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - TYPED_LOOPCHECK(int8_t, pOutput, p, pCtx->size, isMin); + int8_t *data = (int8_t *)pOutput; + + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - TYPED_LOOPCHECK(int16_t, pOutput, p, pCtx->size, isMin); + int16_t *data = (int16_t *)pOutput; + + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *pData = p; - int32_t *retVal = pCtx->aOutputBuf; + int32_t *data = (int32_t *)pOutput; +#if defined(_DEBUG_VIEW) + pTrace("max value updated according to pre-cal:%d", *data); +#endif - for (int32_t i = 0; i < pCtx->size; ++i) { - *retVal = ((*retVal < pData[i]) ^ isMin) ? pData[i] : *retVal; + if ((*data < val) ^ isMin) { + *data = val; + for (int32_t i = 0; i < (pCtx)->tagInfo.numOfTagCols; ++i) { + SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; + if (__ctx->functionId == TSDB_FUNC_TAG_DUMMY) { + aAggs[TSDB_FUNC_TAG].xFunction(__ctx); + } else if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { + *((int64_t *)__ctx->aOutputBuf) = pCtx->ptsList[index]; + } + } } } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - TYPED_LOOPCHECK(int64_t, pOutput, p, pCtx->size, isMin); + int64_t *data = (int64_t *)pOutput; + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin); } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - TYPED_LOOPCHECK(double, pOutput, p, pCtx->size, isMin); + double *data = (double *)pOutput; + double val = GET_DOUBLE_VAL(tval); + + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - TYPED_LOOPCHECK(float, pOutput, p, pCtx->size, isMin); + float *data = (float *)pOutput; + double val = GET_DOUBLE_VAL(tval); + + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin); } + + return; } - return true; -} + void *p = GET_INPUT_CHAR(pCtx); + *notNullElems = 0; + + if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { + if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { + TYPED_LOOPCHECK_N(int8_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { + TYPED_LOOPCHECK_N(int16_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { + int32_t *pData = p; + int32_t *retVal = pOutput; -static void min_function_setup(SQLFunctionCtx *pCtx) { - void *retVal = pCtx->aOutputBuf; - memset(retVal, 0, pCtx->outputBytes); + for (int32_t i = 0; i < pCtx->size; ++i) { + if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { + continue; + } - int32_t type = 0; - if (pCtx->inputType == TSDB_DATA_TYPE_BINARY) { - type = pCtx->outputType; - } else { - type = pCtx->inputType; + if ((*retVal < pData[i]) ^ isMin) { + *retVal = pData[i]; + DO_UPDATE_TAG_COLUMNS(pCtx, pCtx->ptsList[i]); + } + + *notNullElems += 1; + } +#if defined(_DEBUG_VIEW) + pTrace("max value updated:%d", *retVal); +#endif + } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { + TYPED_LOOPCHECK_N(int64_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); + } + } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { + TYPED_LOOPCHECK_N(double, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { + TYPED_LOOPCHECK_N(float, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); + } +} + +static bool min_func_setup(SQLFunctionCtx *pCtx) { + if (!function_setup(pCtx)) { + return false; // not initialized since it has been initialized } + GET_TRUE_DATA_TYPE(); + switch (type) { case TSDB_DATA_TYPE_INT: - *((int32_t *)retVal) = INT32_MAX; + *((int32_t *)pCtx->aOutputBuf) = INT32_MAX; break; case TSDB_DATA_TYPE_FLOAT: - *((float *)retVal) = FLT_MAX; + *((float *)pCtx->aOutputBuf) = FLT_MAX; break; case TSDB_DATA_TYPE_DOUBLE: - *((double *)retVal) = DBL_MAX; + *((double *)pCtx->aOutputBuf) = DBL_MAX; break; case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)retVal) = INT64_MAX; + *((int64_t *)pCtx->aOutputBuf) = INT64_MAX; break; case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)retVal) = INT16_MAX; + *((int16_t *)pCtx->aOutputBuf) = INT16_MAX; break; case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)retVal) = INT8_MAX; + *((int8_t *)pCtx->aOutputBuf) = INT8_MAX; break; default: pError("illegal data type:%d in min/max query", pCtx->inputType); } - INIT_VAL(pCtx); + return true; } -static void max_function_setup(SQLFunctionCtx *pCtx) { - void *retVal = pCtx->aOutputBuf; - memset(retVal, 0, pCtx->outputBytes); +static bool max_func_setup(SQLFunctionCtx *pCtx) { + if (!function_setup(pCtx)) { + return false; // not initialized since it has been initialized + } - int32_t type = (pCtx->inputType == TSDB_DATA_TYPE_BINARY) ? pCtx->outputType : pCtx->inputType; + GET_TRUE_DATA_TYPE(); switch (type) { case TSDB_DATA_TYPE_INT: - *((int32_t *)retVal) = INT32_MIN; + *((int32_t *)pCtx->aOutputBuf) = INT32_MIN; break; case TSDB_DATA_TYPE_FLOAT: - *((float *)retVal) = -FLT_MIN; + *((float *)pCtx->aOutputBuf) = -FLT_MIN; break; case TSDB_DATA_TYPE_DOUBLE: - *((double *)retVal) = -DBL_MIN; + *((double *)pCtx->aOutputBuf) = -DBL_MIN; break; case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)retVal) = INT64_MIN; + *((int64_t *)pCtx->aOutputBuf) = INT64_MIN; break; case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)retVal) = INT16_MIN; + *((int16_t *)pCtx->aOutputBuf) = INT16_MIN; break; case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)retVal) = INT8_MIN; + *((int8_t *)pCtx->aOutputBuf) = INT8_MIN; break; default: pError("illegal data type:%d in min/max query", pCtx->inputType); } - INIT_VAL(pCtx); -} - -static bool min_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - bool ret = minMax_function(pCtx, pCtx->aOutputBuf, 1, ¬NullElems); - - SET_VAL(pCtx, notNullElems, 1); - return ret; -} - -static bool max_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - bool ret = minMax_function(pCtx, pCtx->aOutputBuf, 0, ¬NullElems); - - SET_VAL(pCtx, notNullElems, 1); - return ret; + return true; } -static bool min_dist_intern_function(SQLFunctionCtx *pCtx) { +/* + * the output result of min/max function is the final output buffer, not the intermediate result buffer + */ +static void min_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; minMax_function(pCtx, pCtx->aOutputBuf, 1, ¬NullElems); SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - pCtx->aOutputBuf[pCtx->inputBytes] = DATA_SET_FLAG; - } + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->hasResult = DATA_SET_FLAG; - return true; + // set the flag for super table query + if (pResInfo->superTableQ) { + *(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG; + } + } } -static bool max_dist_intern_function(SQLFunctionCtx *pCtx) { +static void max_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; minMax_function(pCtx, pCtx->aOutputBuf, 0, ¬NullElems); SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - pCtx->aOutputBuf[pCtx->inputBytes] = DATA_SET_FLAG; - } + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->hasResult = DATA_SET_FLAG; - return true; + // set the flag for super table query + if (pResInfo->superTableQ) { + *(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG; + } + } } -static int32_t minmax_dist_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { +static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { int32_t notNullElems = 0; - int32_t type = (pCtx->inputType != TSDB_DATA_TYPE_BINARY) ? pCtx->inputType : pCtx->outputType; + GET_TRUE_DATA_TYPE(); + + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pResInfo->superTableQ); + for (int32_t i = 0; i < pCtx->size; ++i) { char *input = GET_INPUT_CHAR_INDEX(pCtx, i); if (input[bytes] != DATA_SET_FLAG) { continue; } - notNullElems++; switch (type) { case TSDB_DATA_TYPE_TINYINT: { - int8_t v = *(int8_t *)input; - if ((*(int8_t *)output < v) ^ isMin) { - *(int8_t *)output = v; - } + int8_t v = GET_INT8_VAL(input); + UPDATE_DATA(pCtx, *(int8_t *)output, v, notNullElems, isMin); break; }; case TSDB_DATA_TYPE_SMALLINT: { - int16_t v = *(int16_t *)input; - if ((*(int16_t *)output < v) ^ isMin) { - *(int16_t *)output = v; - } + int16_t v = GET_INT16_VAL(input); + UPDATE_DATA(pCtx, *(int16_t *)output, v, notNullElems, isMin); break; } case TSDB_DATA_TYPE_INT: { - int32_t v = *(int32_t *)input; + int32_t v = GET_INT32_VAL(input); if ((*(int32_t *)output < v) ^ isMin) { *(int32_t *)output = v; + + for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { + SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; + aAggs[TSDB_FUNC_TAG].xFunction(__ctx); + } + + notNullElems++; } break; } case TSDB_DATA_TYPE_FLOAT: { - float v = *(float *)input; - if ((*(float *)output < v) ^ isMin) { - *(float *)output = v; - } + float v = GET_FLOAT_VAL(input); + UPDATE_DATA(pCtx, *(float *)output, v, notNullElems, isMin); break; } case TSDB_DATA_TYPE_DOUBLE: { - double v = *(double *)input; - if ((*(double *)output < v) ^ isMin) { - *(double *)output = v; - } + double v = GET_DOUBLE_VAL(input); + UPDATE_DATA(pCtx, *(double *)output, v, notNullElems, isMin); break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t v = *(int64_t *)input; - if ((*(int64_t *)output < v) ^ isMin) { - *(int64_t *)output = v; - } + int64_t v = GET_INT64_VAL(input); + UPDATE_DATA(pCtx, *(int64_t *)output, v, notNullElems, isMin); break; }; default: @@ -1006,314 +1143,352 @@ static int32_t minmax_dist_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char return notNullElems; } -static void min_dist_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = minmax_dist_merge_impl(pCtx, pCtx->inputBytes, pCtx->aOutputBuf, 1); +static void min_func_merge(SQLFunctionCtx *pCtx) { + int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->inputBytes, pCtx->aOutputBuf, 1); SET_VAL(pCtx, notNullElems, 1); - if (notNullElems > 0) { - (pCtx->aOutputBuf)[pCtx->inputBytes] = DATA_SET_FLAG; - pCtx->numOfIteratedElems += notNullElems; + + if (notNullElems > 0) { // for super table query, SResultInfo is not used + char *flag = pCtx->aOutputBuf + pCtx->inputBytes; + *flag = DATA_SET_FLAG; } } -static void min_dist_second_merge(SQLFunctionCtx *pCtx) { - char * output = (char *)pCtx->aOutputBuf; - int32_t notNullElems = minmax_dist_merge_impl(pCtx, pCtx->outputBytes, output, 1); +static void min_func_second_merge(SQLFunctionCtx *pCtx) { + int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 1); SET_VAL(pCtx, notNullElems, 1); + + SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (notNullElems > 0) { - pCtx->numOfIteratedElems += notNullElems; + pResInfo->hasResult = DATA_SET_FLAG; } } -static void max_dist_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = minmax_dist_merge_impl(pCtx, pCtx->inputBytes, pCtx->aOutputBuf, 0); +static void max_func_merge(SQLFunctionCtx *pCtx) { + int32_t numOfElems = minmax_merge_impl(pCtx, pCtx->inputBytes, pCtx->aOutputBuf, 0); - SET_VAL(pCtx, notNullElems, 1); - if (notNullElems > 0) { - (pCtx->aOutputBuf)[pCtx->inputBytes] = DATA_SET_FLAG; - pCtx->numOfIteratedElems += notNullElems; + SET_VAL(pCtx, numOfElems, 1); + if (numOfElems > 0) { + char *flag = pCtx->aOutputBuf + pCtx->inputBytes; + *flag = DATA_SET_FLAG; } } -static void max_dist_second_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = minmax_dist_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 0); +static void max_func_second_merge(SQLFunctionCtx *pCtx) { + int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 0); - SET_VAL(pCtx, notNullElems, 1); - if (notNullElems > 0) { - pCtx->numOfIteratedElems += notNullElems; + SET_VAL(pCtx, numOfElem, 1); + + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + if (numOfElem > 0) { + pResInfo->hasResult = DATA_SET_FLAG; } } -static bool minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin) { +static void minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + int32_t num = 0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { int8_t *output = (int8_t *)pCtx->aOutputBuf; - int8_t i = *(int8_t *)pData; - *output = ((*output < i) ^ isMin) ? i : *output; + int8_t i = GET_INT8_VAL(pData); + + UPDATE_DATA(pCtx, *output, i, num, isMin); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { int16_t *output = pCtx->aOutputBuf; - int16_t i = *(int16_t *)pData; - *output = ((*output < i) ^ isMin) ? i : *output; + int16_t i = GET_INT16_VAL(pData); + + UPDATE_DATA(pCtx, *output, i, num, isMin); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { int32_t *output = pCtx->aOutputBuf; - int32_t i = *(int32_t *)pData; - *output = ((*output < i) ^ isMin) ? i : *output; + int32_t i = GET_INT32_VAL(pData); + + UPDATE_DATA(pCtx, *output, i, num, isMin); } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { int64_t *output = pCtx->aOutputBuf; - int64_t i = *(int64_t *)pData; - *output = ((*output < i) ^ isMin) ? i : *output; + int64_t i = GET_INT64_VAL(pData); + + UPDATE_DATA(pCtx, *output, i, num, isMin); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { float *output = pCtx->aOutputBuf; - float i = *(float *)pData; - *output = ((*output < i) ^ isMin) ? i : *output; + float i = GET_FLOAT_VAL(pData); + + UPDATE_DATA(pCtx, *output, i, num, isMin); } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { double *output = pCtx->aOutputBuf; - double i = *(double *)pData; - *output = ((*output < i) ^ isMin) ? i : *output; + double i = GET_DOUBLE_VAL(pData); + + UPDATE_DATA(pCtx, *output, i, num, isMin); } - return true; + GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } -static bool min_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } SET_VAL(pCtx, 1, 1); - return minMax_function_f(pCtx, index, 1); + minMax_function_f(pCtx, index, 0); + + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult == DATA_SET_FLAG) { + char *flag = pCtx->aOutputBuf + pCtx->inputBytes; + *flag = DATA_SET_FLAG; + } } -static bool max_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } SET_VAL(pCtx, 1, 1); - return minMax_function_f(pCtx, index, 0); -} + minMax_function_f(pCtx, index, 1); -static bool min_dist_intern_function_f(SQLFunctionCtx *pCtx, int32_t index) { - min_function_f(pCtx, index); - if (pCtx->numOfIteratedElems) { - (pCtx->aOutputBuf)[pCtx->inputBytes] = DATA_SET_FLAG; + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult == DATA_SET_FLAG) { + char *flag = pCtx->aOutputBuf + pCtx->inputBytes; + *flag = DATA_SET_FLAG; } - - return true; } -static bool max_dist_intern_function_f(SQLFunctionCtx *pCtx, int32_t index) { - max_function_f(pCtx, index); - if (pCtx->numOfIteratedElems) { - ((char *)pCtx->aOutputBuf)[pCtx->inputBytes] = DATA_SET_FLAG; +#define LOOP_STDDEV_IMPL(type, r, d, ctx, delta, tsdbType) \ + for (int32_t i = 0; i < (ctx)->size; ++i) { \ + if ((ctx)->hasNull && isNull((char *)&((type *)d)[i], tsdbType)) { \ + continue; \ + } \ + (r) += POW2(((type *)d)[i] - (delta)); \ } - return true; -} - -#define LOOP_STDDEV_IMPL(type, r, d, n, delta, tsdbType) \ - for (int32_t i = 0; i < (n); ++i) { \ - if (isNull((char *)&((type *)d)[i], tsdbType)) { \ - continue; \ - } \ - (r) += tPow(((type *)d)[i] - (delta)); \ - } +static void stddev_function(SQLFunctionCtx *pCtx) { + // the second stage to calculate standard deviation + SStddevInfo *pStd = GET_RES_INFO(pCtx)->interResultBuf; -static bool stddev_function(SQLFunctionCtx *pCtx) { - if (pCtx->currentStage == 0) { - /* the first stage to calculate average value */ - return sum_function(pCtx); + if (pStd->stage == 0) { // the first stage is to calculate average value + avg_function(pCtx); } else { - /* the second stage to calculate standard deviation */ - double *retVal = pCtx->aOutputBuf; - double avg = pCtx->intermediateBuf[1].dKey; - void * pData = GET_INPUT_CHAR(pCtx); + double *retVal = &pStd->res; + double avg = pStd->avg; + + void *pData = GET_INPUT_CHAR(pCtx); switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { for (int32_t i = 0; i < pCtx->size; ++i) { - if (isNull(&((int32_t *)pData)[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull(&((int32_t *)pData)[i], pCtx->inputType)) { continue; } - *retVal += tPow(((int32_t *)pData)[i] - avg); + *retVal += POW2(((int32_t *)pData)[i] - avg); } break; } case TSDB_DATA_TYPE_FLOAT: { - LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx->size, avg, pCtx->inputType); + LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType); break; } case TSDB_DATA_TYPE_DOUBLE: { - LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx->size, avg, pCtx->inputType); + LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType); break; } case TSDB_DATA_TYPE_BIGINT: { - LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx->size, avg, pCtx->inputType); + LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType); break; } case TSDB_DATA_TYPE_SMALLINT: { - LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx->size, avg, pCtx->inputType); + LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType); break; } case TSDB_DATA_TYPE_TINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx->size, avg, pCtx->inputType); + LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType); break; } default: pError("stddev function not support data type:%d", pCtx->inputType); } - return true; + // TODO get the correct data + SET_VAL(pCtx, 1, 1); } } -static bool stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { - if (pCtx->currentStage == 0) { - /* the first stage is to calculate average value */ - return sum_function_f(pCtx, index); +static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { + // the second stage to calculate standard deviation + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SStddevInfo *pStd = pResInfo->interResultBuf; + + /* the first stage is to calculate average value */ + if (pStd->stage == 0) { + avg_function_f(pCtx, index); } else { - /* the second stage to calculate standard deviation */ - double *retVal = pCtx->aOutputBuf; - double avg = pCtx->intermediateBuf[1].dKey; + double avg = pStd->avg; + void * pData = GET_INPUT_CHAR_INDEX(pCtx, index); - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { - *retVal += tPow((*(int32_t *)pData) - avg); + pStd->res += POW2(GET_INT32_VAL(pData) - avg); break; } case TSDB_DATA_TYPE_FLOAT: { - *retVal += tPow((*(float *)pData) - avg); + pStd->res += POW2(GET_FLOAT_VAL(pData) - avg); break; } case TSDB_DATA_TYPE_DOUBLE: { - *retVal += tPow((*(double *)pData) - avg); + pStd->res += POW2(GET_DOUBLE_VAL(pData) - avg); break; } case TSDB_DATA_TYPE_BIGINT: { - *retVal += tPow((*(int64_t *)pData) - avg); + pStd->res += POW2(GET_INT64_VAL(pData) - avg); break; } case TSDB_DATA_TYPE_SMALLINT: { - *retVal += tPow((*(int16_t *)pData) - avg); + pStd->res += POW2(GET_INT16_VAL(pData) - avg); break; } case TSDB_DATA_TYPE_TINYINT: { - *retVal += tPow((*(int8_t *)pData) - avg); + pStd->res += POW2(GET_INT8_VAL(pData) - avg); break; } default: pError("stddev function not support data type:%d", pCtx->inputType); } - return true; + SET_VAL(pCtx, 1, 1); } } -static bool stddev_next_step(SQLFunctionCtx *pCtx) { - if (pCtx->currentStage == 0) { +static void stddev_next_step(SQLFunctionCtx *pCtx) { + /* + * the stddevInfo and the average info struct share the same buffer area + * And the position of each element in their struct is exactly the same matched + */ + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SStddevInfo *pStd = pResInfo->interResultBuf; + + if (pStd->stage == 0) { /* * stddev is calculated in two stage: - * 1. get the average value of all points; + * 1. get the average value of all data; * 2. get final result, based on the average values; * so, if this routine is in second stage, no further step is required */ - - ++pCtx->currentStage; + pStd->stage++; avg_finalizer(pCtx); // save average value into tmpBuf, for second stage scan - pCtx->intermediateBuf[1].dKey = ((double *)pCtx->aOutputBuf)[0]; - *((double *)pCtx->aOutputBuf) = 0; - return true; + SAvgInfo *pAvg = pResInfo->interResultBuf; + + pStd->avg = GET_DOUBLE_VAL(pCtx->aOutputBuf); + assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); + } else { + pResInfo->complete = true; } - return false; } static void stddev_finalizer(SQLFunctionCtx *pCtx) { - if (pCtx->numOfIteratedElems <= 0) { + SStddevInfo *pStd = (SStddevInfo *)GET_RES_INFO(pCtx)->interResultBuf; + + if (pStd->num <= 0) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); - return; + } else { + double *retValue = (double *)pCtx->aOutputBuf; + *retValue = sqrt(pStd->res / pStd->num); + SET_VAL(pCtx, 1, 1); + } + + resetResultInfo(GET_RES_INFO(pCtx)); +} + +////////////////////////////////////////////////////////////////////////////////////// +static bool first_last_function_setup(SQLFunctionCtx *pCtx) { + if (!function_setup(pCtx)) { + return false; } - double *retValue = (double *)pCtx->aOutputBuf; - *retValue = sqrt(*retValue / pCtx->numOfIteratedElems); + // used to keep the timestamp for comparison + pCtx->param[1].nType = 0; + pCtx->param[1].i64Key = 0; + + return true; } -static bool first_function(SQLFunctionCtx *pCtx) { +// todo opt for null block +static void first_function(SQLFunctionCtx *pCtx) { if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) || pCtx->order == TSQL_SO_DESC) { - return true; + return; } int32_t notNullElems = 0; - if (pCtx->hasNullValue) { - // handle the null value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); - if (isNull(data, pCtx->inputType)) { - continue; - } - - memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); - notNullElems++; - break; + // handle the null value + for (int32_t i = 0; i < pCtx->size; ++i) { + char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + continue; } - } else { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, 0); - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - notNullElems = pCtx->size; + memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); + DO_UPDATE_TAG_COLUMNS(pCtx, i); + + SResultInfo *pInfo = GET_RES_INFO(pCtx); + pInfo->hasResult = DATA_SET_FLAG; + pInfo->complete = true; + + notNullElems++; + break; } SET_VAL(pCtx, notNullElems, 1); - return notNullElems <= 0; } -static bool first_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->order == TSQL_SO_DESC) { - return true; + return; } void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } SET_VAL(pCtx, 1, 1); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + DO_UPDATE_TAG_COLUMNS(pCtx, 0); - // no need to re-enter current data block - return false; + SResultInfo *pInfo = GET_RES_INFO(pCtx); + pInfo->hasResult = DATA_SET_FLAG; + pInfo->complete = true; // get the first not-null data, completed } static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { - char * retVal = pCtx->aOutputBuf; int64_t *timestamp = pCtx->ptsList; - if (IS_DATA_NOT_ASSIGNED(pCtx) || timestamp[index] < *(int64_t *)retVal) { - *((int64_t *)retVal) = timestamp[index]; - retVal[TSDB_KEYSIZE] = DATA_SET_FLAG; - memcpy(&retVal[TSDB_KEYSIZE + DATA_SET_FLAG_SIZE], pData, pCtx->inputBytes); - SET_DATA_ASSIGNED(pCtx); + SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); + + if (pInfo->hasResult != DATA_SET_FLAG || timestamp[index] < pInfo->ts) { + memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + pInfo->hasResult = DATA_SET_FLAG; + pInfo->ts = timestamp[index]; + + DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); } } /* - * format of intermediate result: "timestamp,value" need to compare the timestamp in the first part (before the comma) to - * decide if the value is earlier than current intermediate result + * format of intermediate result: "timestamp,value" need to compare the timestamp in the first part (before the comma) + * to decide if the value is earlier than current intermediate result */ -static bool first_dist_function(SQLFunctionCtx *pCtx) { +static void first_dist_function(SQLFunctionCtx *pCtx) { if (pCtx->size == 0) { - return true; + return; } /* @@ -1322,171 +1497,164 @@ static bool first_dist_function(SQLFunctionCtx *pCtx) { * 2. scan data files in desc order */ if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) || pCtx->order == TSQL_SO_DESC) { - return true; + return; } int32_t notNullElems = 0; - if (pCtx->hasNullValue) { - int32_t i = 0; - // find the first not null value - while (i < pCtx->size) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); - if (!isNull(data, pCtx->inputType)) { - break; - } - i++; + // find the first not null value + for (int32_t i = 0; i < pCtx->size; ++i) { + char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + continue; } - if (i < pCtx->size) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); - first_data_assign_impl(pCtx, pData, i); - notNullElems++; - } else { - // no data, all data are null - // do nothing - } - } else { - char *pData = GET_INPUT_CHAR(pCtx); - first_data_assign_impl(pCtx, pData, 0); - notNullElems = pCtx->size; + first_data_assign_impl(pCtx, data, i); + + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->hasResult = DATA_SET_FLAG; + + notNullElems++; + break; } SET_VAL(pCtx, notNullElems, 1); - return true; } -static bool first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->size == 0) { - return true; + return; } char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } if (pCtx->order == TSQL_SO_DESC) { - return true; + return; } - first_data_assign_impl(pCtx, pData, 0); + first_data_assign_impl(pCtx, pData, index); SET_VAL(pCtx, 1, 1); - return true; } -static void first_dist_merge(SQLFunctionCtx *pCtx) { +static void first_dist_func_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - if (pData[TSDB_KEYSIZE] != DATA_SET_FLAG) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pCtx->size == 1 && pResInfo->superTableQ); + + SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); + if (pInput->hasResult != DATA_SET_FLAG) { return; } - if (IS_DATA_NOT_ASSIGNED(pCtx) || *(int64_t *)pData < *(int64_t *)pCtx->aOutputBuf) { - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + TSDB_KEYSIZE + DATA_SET_FLAG_SIZE); - SET_DATA_ASSIGNED(pCtx); + SFirstLastInfo *pOutput = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); + if (pOutput->hasResult != DATA_SET_FLAG || pInput->ts < pOutput->ts) { + memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + sizeof(SFirstLastInfo)); + DO_UPDATE_TAG_COLUMNS(pCtx, 0); } - - pCtx->numOfIteratedElems += 1; } -static void first_dist_second_merge(SQLFunctionCtx *pCtx) { - char *pData = GET_INPUT_CHAR(pCtx); +static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { + assert(pCtx->resultInfo->superTableQ); - if (pData[TSDB_KEYSIZE] != DATA_SET_FLAG) { + char * pData = GET_INPUT_CHAR(pCtx); + SFirstLastInfo *pInput = (pData + pCtx->outputBytes); + if (pInput->hasResult != DATA_SET_FLAG) { return; } - /* - * NOTE: if secondary merge is not continue executed, the detection of if data assigned or not may be failed. - * Execution on other tables may change the value, since the SQLFunctionCtx is shared by all tables belonged - * to different groups - */ - if (pCtx->intermediateBuf[0].i64Key == 0 || *(int64_t *)pData < pCtx->intermediateBuf[0].i64Key) { - pCtx->intermediateBuf[0].i64Key = *(int64_t *)pData; - memcpy(pCtx->aOutputBuf, pData + TSDB_KEYSIZE + DATA_SET_FLAG_SIZE, pCtx->outputBytes); + // The param[1] is used to keep the initial value of max ts value + if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i64Key > pInput->ts) { + memcpy(pCtx->aOutputBuf, pData, pCtx->outputBytes); + pCtx->param[1].i64Key = pInput->ts; + pCtx->param[1].nType = pCtx->outputType; + + DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } SET_VAL(pCtx, 1, 1); + GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } ////////////////////////////////////////////////////////////////////////////////////////// /* - * - * last function problem: + * last function: * 1. since the last block may be all null value, so, we simply access the last block is not valid * each block need to be checked. - * 2. If numOfNullPoints == pBlock->numOfBlocks, the whole block is empty. Otherwise, there is at - * least one data in this block that is not null. - * 3. we access the data block in ascending order, so comparison is not needed. The later accessed - * block must have greater value of timestamp. + * 2. If numOfNull == pBlock->numOfBlocks, the whole block is empty. Otherwise, there is at + * least one data in this block that is not null.(TODO opt for this case) */ -static bool last_function(SQLFunctionCtx *pCtx) { +static void last_function(SQLFunctionCtx *pCtx) { if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) || pCtx->order == TSQL_SO_ASC) { - return true; + return; } int32_t notNullElems = 0; - if (pCtx->hasNullValue) { - /* get the last not NULL records */ - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); - if (isNull(data, pCtx->inputType)) { - continue; - } - - memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); - notNullElems++; - break; + for (int32_t i = pCtx->size - 1; i >= 0; --i) { + char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + continue; } - } else { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, pCtx->size - 1); - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - notNullElems = pCtx->size; + memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); + DO_UPDATE_TAG_COLUMNS(pCtx, 0); + + SResultInfo *pInfo = GET_RES_INFO(pCtx); + pInfo->hasResult = DATA_SET_FLAG; + + pInfo->complete = true; // set query completed on this column + notNullElems++; + break; } SET_VAL(pCtx, notNullElems, 1); - return notNullElems <= 0; } -static bool last_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->order == TSQL_SO_ASC) { - return true; + return; } void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } SET_VAL(pCtx, 1, 1); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + DO_UPDATE_TAG_COLUMNS(pCtx, 0); - // no need to re-enter current data block - return false; + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->hasResult = DATA_SET_FLAG; + pResInfo->complete = true; // set query completed } static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { - char * retVal = pCtx->aOutputBuf; int64_t *timestamp = pCtx->ptsList; - if (IS_DATA_NOT_ASSIGNED(pCtx) || *(int64_t *)retVal < timestamp[index]) { + SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); + + if (pInfo->hasResult != DATA_SET_FLAG || pInfo->ts < timestamp[index]) { #if defined(_DEBUG_VIEW) pTrace("assign index:%d, ts:%lld, val:%d, ", index, timestamp[index], *(int32_t *)pData); #endif - *((int64_t *)retVal) = timestamp[index]; - retVal[TSDB_KEYSIZE] = DATA_SET_FLAG; - memcpy(&retVal[TSDB_KEYSIZE + DATA_SET_FLAG_SIZE], pData, pCtx->inputBytes); - SET_DATA_ASSIGNED(pCtx); + + memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + pInfo->hasResult = DATA_SET_FLAG; + pInfo->ts = timestamp[index]; + + DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); } } -static bool last_dist_function(SQLFunctionCtx *pCtx) { +static void last_dist_function(SQLFunctionCtx *pCtx) { if (pCtx->size == 0) { - return true; + return; } /* @@ -1494,46 +1662,37 @@ static bool last_dist_function(SQLFunctionCtx *pCtx) { * 2. for data blocks that are not loaded, no need to check data */ if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) || pCtx->order == TSQL_SO_ASC) { - return true; + return; } int32_t notNullElems = 0; - if (pCtx->hasNullValue) { - int32_t i = pCtx->size - 1; - while (i >= 0) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); - if (!isNull(data, pCtx->inputType)) { - break; - } - i--; + for (int32_t i = pCtx->size - 1; i >= 0; --i) { + char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + continue; } - if (i < 0) { - /* all data in current block are NULL, do nothing */ - } else { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); - last_data_assign_impl(pCtx, pData, i); - notNullElems++; - } - } else { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, pCtx->size - 1); - last_data_assign_impl(pCtx, pData, pCtx->size - 1); - notNullElems = pCtx->size; + last_data_assign_impl(pCtx, data, i); + + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->hasResult = DATA_SET_FLAG; + + notNullElems++; + break; } SET_VAL(pCtx, notNullElems, 1); - return true; } -static bool last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->size == 0) { - return true; + return; } char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } /* @@ -1541,360 +1700,242 @@ static bool last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { * 2. for data blocks that are not loaded, no need to check data */ if (pCtx->order == TSQL_SO_ASC) { - return true; + return; } last_data_assign_impl(pCtx, pData, index); SET_VAL(pCtx, 1, 1); - return true; } -static void last_dist_merge(SQLFunctionCtx *pCtx) { +static void last_dist_func_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - char *retVal = pCtx->aOutputBuf; - /* the input data is null */ - if (pData[TSDB_KEYSIZE] != DATA_SET_FLAG) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pCtx->size == 1 && pResInfo->superTableQ); + + // the input data is null + SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); + if (pInput->hasResult != DATA_SET_FLAG) { return; } - if (IS_DATA_NOT_ASSIGNED(pCtx) || *(int64_t *)pData > *(int64_t *)retVal) { - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + DATA_SET_FLAG_SIZE + TSDB_KEYSIZE); - SET_DATA_ASSIGNED(pCtx); - } + SFirstLastInfo *pOutput = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); + if (pOutput->hasResult != DATA_SET_FLAG || pOutput->ts < pInput->ts) { + memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + sizeof(SFirstLastInfo)); - pCtx->numOfIteratedElems += 1; + DO_UPDATE_TAG_COLUMNS(pCtx, 0); + } } /* * in the secondary merge(local reduce), the output is limited by the - * final output size, so the main difference between last_dist_merge and second_merge + * final output size, so the main difference between last_dist_func_merge and second_merge * is: the output data format in computing */ -static void last_dist_second_merge(SQLFunctionCtx *pCtx) { +static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - char *retVal = pCtx->aOutputBuf; - /* the input data is null */ - if (pData[TSDB_KEYSIZE] != DATA_SET_FLAG) { + SFirstLastInfo *pInput = (pData + pCtx->outputBytes); + if (pInput->hasResult != DATA_SET_FLAG) { return; } - // todo refactor, pls refer to first_dist_second_merge for reasons - if (pCtx->intermediateBuf[0].i64Key == 0 || *(int64_t *)pData > pCtx->intermediateBuf[0].i64Key) { - pCtx->intermediateBuf[0].i64Key = *(int64_t *)pData; - memcpy(retVal, pData + TSDB_KEYSIZE + DATA_SET_FLAG_SIZE, pCtx->outputBytes); + /* + * param[1] used to keep the corresponding timestamp to decide if current result is + * the true last result + */ + if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i64Key < pInput->ts) { + memcpy(pCtx->aOutputBuf, pData, pCtx->outputBytes); + pCtx->param[1].i64Key = pInput->ts; + pCtx->param[1].nType = pCtx->outputType; + + DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } SET_VAL(pCtx, 1, 1); + GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } +////////////////////////////////////////////////////////////////////////////////// /* - * there must be data in last_row_dist function + * NOTE: last_row does not use the interResultBuf to keep the result */ -static int32_t last_row_dist_function(SQLFunctionCtx *pCtx) { +static void last_row_function(SQLFunctionCtx *pCtx) { assert(pCtx->size == 1); char *pData = GET_INPUT_CHAR(pCtx); - *(TSKEY *)pCtx->aOutputBuf = pCtx->intermediateBuf[1].i64Key; + assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); - SET_DATA_ASSIGNED(pCtx); - assignVal(pCtx->aOutputBuf + TSDB_KEYSIZE + DATA_SET_FLAG_SIZE, pData, pCtx->inputBytes, pCtx->inputType); + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + + SLastrowInfo *pInfo = (SLastrowInfo *)pResInfo->interResultBuf; + pInfo->ts = pCtx->param[0].i64Key; + pInfo->hasResult = DATA_SET_FLAG; + + // set the result to final result buffer + if (pResInfo->superTableQ) { + SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); + pInfo1->ts = pCtx->param[0].i64Key; + pInfo1->hasResult = DATA_SET_FLAG; + + DO_UPDATE_TAG_COLUMNS(pCtx, 0); + } SET_VAL(pCtx, pCtx->size, 1); +} - return true; +static void last_row_finalizer(SQLFunctionCtx *pCtx) { + // do nothing at the first stage + if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult != DATA_SET_FLAG) { + setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + return; + } + } else { + // do nothing + } + + GET_RES_INFO(pCtx)->numOfRes = 1; + resetResultInfo(GET_RES_INFO(pCtx)); } ////////////////////////////////////////////////////////////////////////////////// +static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int64_t tsKey, char *pTags, + SExtTagsInfo *pTagInfo, int16_t stage) { + dst->v.nType = type; + dst->v.i64Key = *(int64_t *)val; + dst->timestamp = tsKey; + + int32_t size = 0; + if (stage == SECONDARY_STAGE_MERGE || stage == FIRST_STAGE_MERGE) { + memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); + } else { // the tags are dumped from the ctx tag fields + for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { + tVariantDump(&pTagInfo->pTagCtxList[i]->tag, dst->pTags + size, pTagInfo->pTagCtxList[i]->tag.nType); + size += pTagInfo->pTagCtxList[i]->outputBytes; + } + } +} -/* - * intermediate parameters usage: - * 1. param[0]: maximum allowable results - * 2. param[1]: order by type (time or value) - * 3. param[2]: asc/desc order - * 4. param[3]: no use - * - * 1. intermediateBuf[0]: number of existed results - * 2. intermediateBuf[1]: results linklist - * 3. intermediateBuf[2]: no use - * 4. intermediateBuf[3]: reserved for tags - * - */ -static void top_bottom_function_setup(SQLFunctionCtx *pCtx) { - /* top-K value */ - pCtx->intermediateBuf[0].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->intermediateBuf[0].i64Key = 0; - - /* - * keep the intermediate results during scan all data blocks - * in the format of: timestamp|value - */ - pCtx->intermediateBuf[1].pz = (tValuePair *)calloc(1, sizeof(tValuePair) * pCtx->param[0].i64Key); - pCtx->intermediateBuf[1].nType = TSDB_DATA_TYPE_BINARY; - - INIT_VAL(pCtx); -} - -#define top_add_elem_impl(list, len, val, ts, seg, mx, type) \ - do { \ - if (len < mx) { \ - if ((len) == 0 || (val) >= list[(len)-1].v.seg) { \ - (list)[len].v.nType = (type); \ - (list)[len].v.seg = (val); \ - (list)[len].timestamp = (ts); \ - } else { \ - int32_t i = (len)-1; \ - while (i >= 0 && (list)[i].v.seg > (val)) { \ - (list)[i + 1] = (list)[i]; \ - i -= 1; \ - } \ - (list)[i + 1].v.nType = (type); \ - (list)[i + 1].v.seg = (val); \ - (list)[i + 1].timestamp = (ts); \ - } \ - len += 1; \ - } else { \ - if ((val) > (list)[0].v.seg) { \ - int32_t i = 0; \ - while (i + 1 < (len) && (list)[i + 1].v.seg < (val)) { \ - (list)[i] = (list)[i + 1]; \ - i += 1; \ - } \ - (list)[i].v.nType = (type); \ - (list)[i].v.seg = (val); \ - (list)[i].timestamp = (ts); \ - } \ - } \ +#define VALUEPAIRASSIGN(dst, src, __l) \ + do { \ + (dst)->timestamp = (src)->timestamp; \ + (dst)->v = (src)->v; \ + memcpy((dst)->pTags, (src)->pTags, (size_t)(__l)); \ } while (0); -#define bottom_add_elem_impl(list, len, val, ts, seg, mx, type) \ - do { \ - if (len < mx) { \ - if ((len) == 0) { \ - (list)[len].v.nType = (type); \ - (list)[len].v.seg = (val); \ - (list)[len].timestamp = (ts); \ - } else { \ - int32_t i = (len)-1; \ - while (i >= 0 && (list)[i].v.seg < (val)) { \ - (list)[i + 1] = (list)[i]; \ - i -= 1; \ - } \ - (list)[i + 1].v.nType = (type); \ - (list)[i + 1].v.seg = (val); \ - (list)[i + 1].timestamp = (ts); \ - } \ - len += 1; \ - } else { \ - if ((val) < (list)[0].v.seg) { \ - int32_t i = 0; \ - while (i + 1 < (len) && (list)[i + 1].v.seg > (val)) { \ - (list)[i] = (list)[i + 1]; \ - i += 1; \ - } \ - (list)[i].v.nType = (type); \ - (list)[i].v.seg = (val); \ - (list)[i].timestamp = (ts); \ - } \ - } \ - } while (0); +static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, + SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { + tVariant val = {0}; + tVariantCreateFromBinary(&val, pData, tDataTypeDesc[type].nSize, type); -static void top_function_do_add(int32_t *len, int32_t maxLen, tValuePair *pList, void *pData, int64_t *timestamp, - uint16_t dataType) { - switch (dataType) { - case TSDB_DATA_TYPE_INT: { - tValuePair *intList = pList; - int32_t value = *(int32_t *)pData; - if (*len < maxLen) { - if (*len == 0 || value >= intList[*len - 1].v.i64Key) { - intList[*len].v.nType = dataType; - intList[*len].v.i64Key = value; - intList[*len].timestamp = *timestamp; - } else { - int32_t i = (*len) - 1; - while (i >= 0 && intList[i].v.i64Key > value) { - intList[i + 1] = intList[i]; - i -= 1; - } - intList[i + 1].v.nType = dataType; - intList[i + 1].v.i64Key = value; - intList[i + 1].timestamp = *timestamp; + tValuePair **pList = pInfo->res; + + if (pInfo->num < maxLen) { + if (pInfo->num == 0 || ((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && + val.i64Key >= pList[pInfo->num - 1]->v.i64Key) || + ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && + val.dKey >= pList[pInfo->num - 1]->v.dKey)) { + valuePairAssign(pList[pInfo->num], type, &val.i64Key, ts, pTags, pTagInfo, stage); + } else { + int32_t i = pInfo->num - 1; + + if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { + while (i >= 0 && pList[i]->v.i64Key > val.i64Key) { + VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); + i -= 1; } - (*len)++; } else { - if (value > pList[0].v.i64Key) { - int32_t i = 0; - while (i + 1 < maxLen && pList[i + 1].v.i64Key < value) { - pList[i] = pList[i + 1]; - i += 1; - } - - pList[i].v.nType = dataType; - pList[i].v.i64Key = value; - pList[i].timestamp = *timestamp; + while (i >= 0 && pList[i]->v.dKey > val.dKey) { + VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); + i -= 1; } } - } break; - case TSDB_DATA_TYPE_DOUBLE: { - top_add_elem_impl(pList, *len, *(double *)pData, *timestamp, dKey, maxLen, dataType); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - top_add_elem_impl(pList, *len, *(int64_t *)pData, *timestamp, i64Key, maxLen, dataType); - break; - } - case TSDB_DATA_TYPE_FLOAT: { - top_add_elem_impl(pList, *len, *(float *)pData, *timestamp, dKey, maxLen, dataType); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - top_add_elem_impl(pList, *len, *(int16_t *)pData, *timestamp, i64Key, maxLen, dataType); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - top_add_elem_impl(pList, *len, *(int8_t *)pData, *timestamp, i64Key, maxLen, dataType); - break; + + valuePairAssign(pList[i + 1], type, &val.i64Key, ts, pTags, pTagInfo, stage); } - default: - pError("top/bottom function not support data type:%d", dataType); - }; -} -static void bottom_function_do_add(int32_t *len, int32_t maxLen, tValuePair *pList, void *pData, int64_t *timestamp, - uint16_t dataType) { - switch (dataType) { - case TSDB_DATA_TYPE_INT: { - int32_t value = *(int32_t *)pData; - if ((*len) < maxLen) { - if (*len == 0) { - pList[*len].v.i64Key = value; - pList[*len].timestamp = *timestamp; - } else { - int32_t i = (*len) - 1; - while (i >= 0 && pList[i].v.i64Key < value) { - pList[i + 1] = pList[i]; - i -= 1; - } - pList[i + 1].v.i64Key = value; - pList[i + 1].timestamp = *timestamp; + pInfo->num++; + } else { + int32_t i = 0; + + if (((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && val.i64Key > pList[0]->v.i64Key) || + ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && val.dKey > pList[0]->v.dKey)) { + // find the appropriate the slot position + if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { + while (i + 1 < maxLen && pList[i + 1]->v.i64Key < val.i64Key) { + VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); + i += 1; } - (*len)++; } else { - if (value < pList[0].v.i64Key) { - int32_t i = 0; - while (i + 1 < maxLen && pList[i + 1].v.i64Key > value) { - pList[i] = pList[i + 1]; - i += 1; - } - pList[i].v.i64Key = value; - pList[i].timestamp = *timestamp; + while (i + 1 < maxLen && pList[i + 1]->v.dKey < val.dKey) { + VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); + i += 1; } } - } break; - case TSDB_DATA_TYPE_DOUBLE: { - bottom_add_elem_impl(pList, *len, *(double *)pData, *timestamp, dKey, maxLen, dataType); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - bottom_add_elem_impl(pList, *len, *(int64_t *)pData, *timestamp, i64Key, maxLen, dataType); - break; - } - case TSDB_DATA_TYPE_FLOAT: { - bottom_add_elem_impl(pList, *len, *(float *)pData, *timestamp, dKey, maxLen, dataType); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - bottom_add_elem_impl(pList, *len, *(int16_t *)pData, *timestamp, i64Key, maxLen, dataType); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - bottom_add_elem_impl(pList, *len, *(int8_t *)pData, *timestamp, i64Key, maxLen, dataType); - break; - } - }; -} - -static bool top_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - if (pCtx->hasNullValue) { - for (int32_t i = 0; i < pCtx->size; ++i) { - if (isNull(GET_INPUT_CHAR_INDEX(pCtx, i), pCtx->inputType)) { - continue; - } - notNullElems++; - top_function_do_add(&pCtx->intermediateBuf[0].i64Key, pCtx->param[0].i64Key, pCtx->intermediateBuf[1].pz, - GET_INPUT_CHAR_INDEX(pCtx, i), &pCtx->ptsList[i], pCtx->inputType); - } - } else { - notNullElems = pCtx->size; - for (int32_t i = 0; i < pCtx->size; ++i) { - top_function_do_add(&pCtx->intermediateBuf[0].i64Key, pCtx->param[0].i64Key, pCtx->intermediateBuf[1].pz, - GET_INPUT_CHAR_INDEX(pCtx, i), &pCtx->ptsList[i], pCtx->inputType); + valuePairAssign(pList[i], type, &val.i64Key, ts, pTags, pTagInfo, stage); } } - - SET_VAL(pCtx, notNullElems, pCtx->intermediateBuf[0].i64Key); - return true; } -static bool top_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; - } - - SET_VAL(pCtx, 1, pCtx->param[0].i64Key); - top_function_do_add(&pCtx->intermediateBuf[0].i64Key, pCtx->param[0].i64Key, pCtx->intermediateBuf[1].pz, pData, - &pCtx->ptsList[index], pCtx->inputType); +static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, + SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { + tValuePair **pList = pInfo->res; - return true; -} + tVariant val = {0}; + tVariantCreateFromBinary(&val, pData, tDataTypeDesc[type].nSize, type); -static bool bottom_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - void * pData = GET_INPUT_CHAR(pCtx); + if (pInfo->num < maxLen) { + if (pInfo->num == 0) { + valuePairAssign(pList[pInfo->num], type, &val.i64Key, ts, pTags, pTagInfo, stage); + } else { + int32_t i = pInfo->num - 1; - if (pCtx->hasNullValue) { - for (int32_t i = 0; i < pCtx->size; ++i) { - if (isNull(GET_INPUT_CHAR_INDEX(pCtx, i), pCtx->inputType)) { - continue; + if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { + while (i >= 0 && pList[i]->v.i64Key < val.i64Key) { + VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); + i -= 1; + } + } else { + while (i >= 0 && pList[i]->v.dKey < val.dKey) { + VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); + i -= 1; + } } - bottom_function_do_add(&pCtx->intermediateBuf[0].i64Key, pCtx->param[0].i64Key, pCtx->intermediateBuf[1].pz, - (char *)pData + pCtx->inputBytes * i, &pCtx->ptsList[i], pCtx->inputType); - notNullElems++; + valuePairAssign(pList[i + 1], type, &val.i64Key, ts, pTags, pTagInfo, stage); } + + pInfo->num++; } else { - notNullElems = pCtx->size; - for (int32_t i = 0; i < pCtx->size; ++i) { - bottom_function_do_add(&pCtx->intermediateBuf[0].i64Key, pCtx->param[0].i64Key, pCtx->intermediateBuf[1].pz, - (char *)pData + pCtx->inputBytes * i, pCtx->ptsList + i, pCtx->inputType); - } - } + int32_t i = 0; - SET_VAL(pCtx, notNullElems, pCtx->intermediateBuf[0].i64Key); - return true; -} + if (((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && val.i64Key < pList[0]->v.i64Key) || + ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && val.dKey < pList[0]->v.dKey)) { + // find the appropriate the slot position + if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { + while (i + 1 < maxLen && pList[i + 1]->v.i64Key > val.i64Key) { + VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); + i += 1; + } + } else { + while (i + 1 < maxLen && pList[i + 1]->v.dKey > val.dKey) { + VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); + i += 1; + } + } -static bool bottom_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + valuePairAssign(pList[i], type, &val.i64Key, ts, pTags, pTagInfo, stage); + } } - - SET_VAL(pCtx, 1, pCtx->param[0].i64Key); - bottom_function_do_add(&pCtx->intermediateBuf[0].i64Key, pCtx->param[0].i64Key, pCtx->intermediateBuf[1].pz, pData, - pCtx->ptsList + index, pCtx->inputType); - - return true; } static int32_t resAscComparFn(const void *pLeft, const void *pRight) { - tValuePair *pLeftElem = (tValuePair *)pLeft; - tValuePair *pRightElem = (tValuePair *)pRight; + tValuePair *pLeftElem = *(tValuePair **)pLeft; + tValuePair *pRightElem = *(tValuePair **)pRight; if (pLeftElem->timestamp == pRightElem->timestamp) { return 0; @@ -1906,8 +1947,8 @@ static int32_t resAscComparFn(const void *pLeft, const void *pRight) { static int32_t resDescComparFn(const void *pLeft, const void *pRight) { return -resAscComparFn(pLeft, pRight); } static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { - tValuePair *pLeftElem = (tValuePair *)pLeft; - tValuePair *pRightElem = (tValuePair *)pRight; + tValuePair *pLeftElem = *(tValuePair **)pLeft; + tValuePair *pRightElem = *(tValuePair **)pRight; int32_t type = pLeftElem->v.nType; if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { @@ -1928,58 +1969,61 @@ static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); } static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { - tValuePair *tvp = (tValuePair *)pCtx->intermediateBuf[1].pz; + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + STopBotInfo *pRes = pResInfo->interResultBuf; - // copy to result set buffer - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); + tValuePair **tvp = pRes->res; + int32_t step = 0; // in case of second stage merge, always use incremental output. if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { step = QUERY_ASC_FORWARD_STEP; + } else { + step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); } - int32_t len = pCtx->numOfOutputElems; + int32_t len = GET_RES_INFO(pCtx)->numOfRes; switch (type) { case TSDB_DATA_TYPE_INT: { int32_t *output = (int32_t *)pCtx->aOutputBuf; for (int32_t i = 0; i < len; ++i, output += step) { - *output = tvp[i].v.i64Key; + *output = tvp[i]->v.i64Key; } break; } case TSDB_DATA_TYPE_BIGINT: { int64_t *output = (int64_t *)pCtx->aOutputBuf; for (int32_t i = 0; i < len; ++i, output += step) { - *output = tvp[i].v.i64Key; + *output = tvp[i]->v.i64Key; } break; } case TSDB_DATA_TYPE_DOUBLE: { double *output = (double *)pCtx->aOutputBuf; for (int32_t i = 0; i < len; ++i, output += step) { - *output = tvp[i].v.dKey; + *output = tvp[i]->v.dKey; } break; } case TSDB_DATA_TYPE_FLOAT: { float *output = (float *)pCtx->aOutputBuf; for (int32_t i = 0; i < len; ++i, output += step) { - *output = tvp[i].v.dKey; + *output = tvp[i]->v.dKey; } break; } case TSDB_DATA_TYPE_SMALLINT: { int16_t *output = (int16_t *)pCtx->aOutputBuf; for (int32_t i = 0; i < len; ++i, output += step) { - *output = tvp[i].v.i64Key; + *output = tvp[i]->v.i64Key; } break; } case TSDB_DATA_TYPE_TINYINT: { int8_t *output = (int8_t *)pCtx->aOutputBuf; for (int32_t i = 0; i < len; ++i, output += step) { - *output = tvp[i].v.i64Key; + *output = tvp[i]->v.i64Key; } break; } @@ -1992,648 +2036,632 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { // set the output timestamp of each record. TSKEY *output = pCtx->ptsOutputBuf; for (int32_t i = 0; i < len; ++i, output += step) { - *output = tvp[i].timestamp; + *output = tvp[i]->timestamp; } -} -static void top_bottom_function_finalizer(SQLFunctionCtx *pCtx) { - /* - * data in temporary list is less than the required count not enough qualified number of results - */ - if (pCtx->intermediateBuf[0].i64Key < pCtx->param[0].i64Key) { - pCtx->numOfOutputElems = pCtx->intermediateBuf[0].i64Key; + // set the corresponding tag data for each record + // todo check malloc failure + char **pData = calloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); + for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { + pData[i] = pCtx->tagInfo.pTagCtxList[i]->aOutputBuf; } - tValuePair *tvp = (tValuePair *)pCtx->intermediateBuf[1].pz; - - // user specify the order of output by sort the result according to timestamp - if (pCtx->param[1].i64Key == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - __compar_fn_t comparator = (pCtx->param[2].i64Key == TSQL_SO_ASC) ? resAscComparFn : resDescComparFn; - qsort(tvp, pCtx->numOfOutputElems, sizeof(tValuePair), comparator); - } else if (pCtx->param[1].i64Key > PRIMARYKEY_TIMESTAMP_COL_INDEX) { - __compar_fn_t comparator = (pCtx->param[2].i64Key == TSQL_SO_ASC) ? resDataAscComparFn : resDataDescComparFn; - qsort(tvp, pCtx->numOfOutputElems, sizeof(tValuePair), comparator); + for (int32_t i = 0; i < len; ++i, output += step) { + int16_t offset = 0; + for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { + memcpy(pData[j], tvp[i]->pTags + offset, (size_t)pCtx->tagInfo.pTagCtxList[j]->outputBytes); + offset += pCtx->tagInfo.pTagCtxList[j]->outputBytes; + pData[j] += pCtx->tagInfo.pTagCtxList[j]->outputBytes; + } } - int32_t type = pCtx->outputType == TSDB_DATA_TYPE_BINARY ? pCtx->inputBytes : pCtx->outputType; - copyTopBotRes(pCtx, type); - - tfree(pCtx->intermediateBuf[1].pz); + tfree(pData); } -typedef struct STopBotRuntime { - int32_t num; - tValuePair res[]; -} STopBotRuntime; - bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *minval, char *maxval) { - int32_t numOfExistsRes = 0; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - numOfExistsRes = (int32_t)pCtx->intermediateBuf[0].i64Key; - } else { - STopBotRuntime *pTpBtRuntime = (STopBotRuntime *)pCtx->aOutputBuf; - numOfExistsRes = pTpBtRuntime->num; - } + STopBotInfo *pTopBotInfo = (STopBotInfo *)GET_RES_INFO(pCtx)->interResultBuf; + + int32_t numOfExistsRes = pTopBotInfo->num; + // required number of results are not reached, continue load data block if (numOfExistsRes < pCtx->param[0].i64Key) { - /* required number of results are not reached, continue load data block */ return true; - } else { // - tValuePair *pRes = pCtx->intermediateBuf[1].pz; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_TOP_DST) { - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - return *(int8_t *)maxval > pRes[0].v.i64Key; - case TSDB_DATA_TYPE_SMALLINT: - return *(int16_t *)maxval > pRes[0].v.i64Key; - case TSDB_DATA_TYPE_INT: - return *(int32_t *)maxval > pRes[0].v.i64Key; - case TSDB_DATA_TYPE_BIGINT: - return *(int64_t *)maxval > pRes[0].v.i64Key; - case TSDB_DATA_TYPE_FLOAT: - return *(float *)maxval > pRes[0].v.dKey; - case TSDB_DATA_TYPE_DOUBLE: - return *(double *)maxval > pRes[0].v.dKey; - default: - return true; - } - } else { - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - return *(int8_t *)minval < pRes[0].v.i64Key; - case TSDB_DATA_TYPE_SMALLINT: - return *(int16_t *)minval < pRes[0].v.i64Key; - case TSDB_DATA_TYPE_INT: - return *(int32_t *)minval < pRes[0].v.i64Key; - case TSDB_DATA_TYPE_BIGINT: - return *(int64_t *)minval < pRes[0].v.i64Key; - case TSDB_DATA_TYPE_FLOAT: - return *(float *)minval < pRes[0].v.dKey; - case TSDB_DATA_TYPE_DOUBLE: - return *(double *)minval < pRes[0].v.dKey; - default: - return true; - } + } + + tValuePair *pRes = pTopBotInfo->res; + + if (functionId == TSDB_FUNC_TOP) { + switch (pCtx->inputType) { + case TSDB_DATA_TYPE_TINYINT: + return GET_INT8_VAL(maxval) > pRes[0].v.i64Key; + case TSDB_DATA_TYPE_SMALLINT: + return GET_INT16_VAL(maxval) > pRes[0].v.i64Key; + case TSDB_DATA_TYPE_INT: + return GET_INT32_VAL(maxval) > pRes[0].v.i64Key; + case TSDB_DATA_TYPE_BIGINT: + return GET_INT64_VAL(maxval) > pRes[0].v.i64Key; + case TSDB_DATA_TYPE_FLOAT: + return GET_FLOAT_VAL(maxval) > pRes[0].v.dKey; + case TSDB_DATA_TYPE_DOUBLE: + return GET_DOUBLE_VAL(maxval) > pRes[0].v.dKey; + default: + return true; + } + } else { + switch (pCtx->inputType) { + case TSDB_DATA_TYPE_TINYINT: + return GET_INT8_VAL(minval) < pRes[0].v.i64Key; + case TSDB_DATA_TYPE_SMALLINT: + return GET_INT16_VAL(minval) < pRes[0].v.i64Key; + case TSDB_DATA_TYPE_INT: + return GET_INT32_VAL(minval) < pRes[0].v.i64Key; + case TSDB_DATA_TYPE_BIGINT: + return GET_INT64_VAL(minval) < pRes[0].v.i64Key; + case TSDB_DATA_TYPE_FLOAT: + return GET_FLOAT_VAL(minval) < pRes[0].v.dKey; + case TSDB_DATA_TYPE_DOUBLE: + return GET_DOUBLE_VAL(minval) < pRes[0].v.dKey; + default: + return true; } } } /* - * intermediate parameters usage: + * Parameters values: * 1. param[0]: maximum allowable results * 2. param[1]: order by type (time or value) * 3. param[2]: asc/desc order - * 4. param[3]: no use - * - * 1. intermediateBuf[0]: number of existed results - * 2. intermediateBuf[1]: results linklist - * 3. intermediateBuf[2]: no use - * 4. intermediateBuf[3]: reserved for tags * + * top/bottom use the intermediate result buffer to keep the intermediate result */ -static void top_bottom_dist_function_setup(SQLFunctionCtx *pCtx) { - /* top-K value */ - pCtx->intermediateBuf[0].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->intermediateBuf[0].i64Key = 0; +static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); - if (pCtx->outputType == TSDB_DATA_TYPE_BINARY) { - STopBotRuntime *pRes = (STopBotRuntime *)pCtx->aOutputBuf; - pCtx->intermediateBuf[1].pz = pRes->res; - pRes->num = 0; + // only the first_stage_merge is directly written data into final output buffer + if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { + return pCtx->aOutputBuf; + } else { // for normal table query and super table at the secondary_stage, result is written to intermediate buffer + return pResInfo->interResultBuf; + } +} - memset(pRes, 0, sizeof(STopBotRuntime) + pCtx->param[0].i64Key * sizeof(tValuePair)); - } else { - /* - * keep the intermediate results during scan all data blocks in the format of: timestamp|value - */ - int32_t size = sizeof(tValuePair) * pCtx->param[0].i64Key; - pCtx->intermediateBuf[1].pz = (tValuePair *)calloc(1, size); - pCtx->intermediateBuf[1].nType = TSDB_DATA_TYPE_BINARY; - pCtx->intermediateBuf[1].nLen = size; +/* + * keep the intermediate results during scan data blocks in the format of: + * +-----------------------------------+-------------one value pair-----------+------------next value pair-----------+ + * |-------------pointer area----------|----ts---+-----+-----n tags-----------|----ts---+-----+-----n tags-----------| + * +..[Value Pointer1][Value Pointer2].|timestamp|value|tags1|tags2|....|tagsn|timestamp|value|tags1|tags2|....|tagsn+ + */ +static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { + char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo); + pTopBotInfo->res = tmp; + + tmp += POINTER_BYTES * pCtx->param[0].i64Key; + + size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; + + for (int32_t i = 0; i < pCtx->param[0].i64Key; ++i) { + pTopBotInfo->res[i] = tmp; + pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); + tmp += size; + } +} + +static bool top_bottom_function_setup(SQLFunctionCtx *pCtx) { + if (!function_setup(pCtx)) { + return false; } - INIT_VAL(pCtx); + STopBotInfo *pInfo = getTopBotOutputInfo(pCtx); + buildTopBotStruct(pInfo, pCtx); + + return true; } -static bool top_dist_function(SQLFunctionCtx *pCtx) { +static void top_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - STopBotRuntime *pRes = (STopBotRuntime *)pCtx->aOutputBuf; - if (pCtx->hasNullValue) { - for (int32_t i = 0; i < pCtx->size; ++i) { - if (isNull(GET_INPUT_CHAR_INDEX(pCtx, i), pCtx->inputType)) { - continue; - } - notNullElems++; - top_function_do_add(&pRes->num, pCtx->param[0].i64Key, pCtx->intermediateBuf[1].pz, GET_INPUT_CHAR_INDEX(pCtx, i), - &pCtx->ptsList[i], pCtx->inputType); - } - } else { - notNullElems = pCtx->size; - for (int32_t i = 0; i < pCtx->size; ++i) { - top_function_do_add(&pRes->num, pCtx->param[0].i64Key, pCtx->intermediateBuf[1].pz, GET_INPUT_CHAR_INDEX(pCtx, i), - &pCtx->ptsList[i], pCtx->inputType); + STopBotInfo *pRes = getTopBotOutputInfo(pCtx); + assert(pRes->num >= 0); + + for (int32_t i = 0; i < pCtx->size; ++i) { + char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + continue; } + + notNullElems++; + do_top_function_add(pRes, pCtx->param[0].i64Key, data, pCtx->ptsList[i], pCtx->inputType, &pCtx->tagInfo, NULL, 0); + } + + if (!pCtx->hasNull) { + assert(pCtx->size == notNullElems); } - /* treat the result as only one result */ + // treat the result as only one result SET_VAL(pCtx, notNullElems, 1); - return true; + + if (notNullElems > 0) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->hasResult = DATA_SET_FLAG; + } } -static bool top_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } - STopBotRuntime *pRes = (STopBotRuntime *)pCtx->aOutputBuf; + STopBotInfo *pRes = getTopBotOutputInfo(pCtx); + assert(pRes->num >= 0); SET_VAL(pCtx, 1, 1); - top_function_do_add(&pRes->num, pCtx->param[0].i64Key, pCtx->intermediateBuf[1].pz, pData, &pCtx->ptsList[index], - pCtx->inputType); + do_top_function_add(pRes, pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL, + 0); - return true; + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->hasResult = DATA_SET_FLAG; } -static void top_dist_merge(SQLFunctionCtx *pCtx) { +static void top_func_merge(SQLFunctionCtx *pCtx) { char *input = GET_INPUT_CHAR(pCtx); - STopBotRuntime *pInput = (STopBotRuntime *)input; + STopBotInfo *pInput = (STopBotInfo *)input; if (pInput->num <= 0) { return; } - STopBotRuntime *pRes = (STopBotRuntime *)pCtx->aOutputBuf; + // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary + buildTopBotStruct(pInput, pCtx); + + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pResInfo->superTableQ && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); + + STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); + for (int32_t i = 0; i < pInput->num; ++i) { - top_function_do_add(&pRes->num, pCtx->param[0].i64Key, pRes->res, &pInput->res[i].v.i64Key, - &pInput->res[i].timestamp, pCtx->inputType); + do_top_function_add(pOutput, pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, + pCtx->inputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } - - pCtx->numOfIteratedElems = pRes->num; } -static void top_dist_second_merge(SQLFunctionCtx *pCtx) { - STopBotRuntime *pInput = (STopBotRuntime *)GET_INPUT_CHAR(pCtx); +static void top_func_second_merge(SQLFunctionCtx *pCtx) { + STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_CHAR(pCtx); + + // construct the input data struct from binary data + buildTopBotStruct(pInput, pCtx); + + STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - /* the intermediate result is binary, we only use the output data type */ + // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { - top_function_do_add(&pCtx->intermediateBuf[0].i64Key, pCtx->param[0].i64Key, pCtx->intermediateBuf[1].pz, - &pInput->res[i].v.i64Key, &pInput->res[i].timestamp, pCtx->outputType); + do_top_function_add(pOutput, pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, + pCtx->outputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } - SET_VAL(pCtx, pInput->num, pCtx->intermediateBuf[0].i64Key); + SET_VAL(pCtx, pInput->num, pOutput->num); + + if (pOutput->num > 0) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->hasResult = DATA_SET_FLAG; + } } -static bool bottom_dist_function(SQLFunctionCtx *pCtx) { +static void bottom_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - STopBotRuntime *pRes = (STopBotRuntime *)pCtx->aOutputBuf; - if (pCtx->hasNullValue) { - for (int32_t i = 0; i < pCtx->size; ++i) { - if (isNull(GET_INPUT_CHAR_INDEX(pCtx, i), pCtx->inputType)) { - continue; - } - notNullElems++; - bottom_function_do_add(&pRes->num, pCtx->param[0].i64Key, pCtx->intermediateBuf[1].pz, - GET_INPUT_CHAR_INDEX(pCtx, i), &pCtx->ptsList[i], pCtx->inputType); - } - } else { - notNullElems = pCtx->size; - for (int32_t i = 0; i < pCtx->size; ++i) { - bottom_function_do_add(&pRes->num, pCtx->param[0].i64Key, pCtx->intermediateBuf[1].pz, - GET_INPUT_CHAR_INDEX(pCtx, i), &pCtx->ptsList[i], pCtx->inputType); + STopBotInfo *pRes = getTopBotOutputInfo(pCtx); + + for (int32_t i = 0; i < pCtx->size; ++i) { + char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + continue; } + + notNullElems++; + do_bottom_function_add(pRes, pCtx->param[0].i64Key, data, pCtx->ptsList[i], pCtx->inputType, &pCtx->tagInfo, NULL, + 0); + } + + if (!pCtx->hasNull) { + assert(pCtx->size == notNullElems); } - /* treat the result as only one result */ + // treat the result as only one result SET_VAL(pCtx, notNullElems, 1); - return true; + + if (notNullElems > 0) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->hasResult = DATA_SET_FLAG; + } } -static bool bottom_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void bottom_function_f(SQLFunctionCtx *pCtx, int32_t index) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } - STopBotRuntime *pRes = (STopBotRuntime *)pCtx->aOutputBuf; - + STopBotInfo *pRes = getTopBotOutputInfo(pCtx); SET_VAL(pCtx, 1, 1); - bottom_function_do_add(&pRes->num, pCtx->param[0].i64Key, pCtx->intermediateBuf[1].pz, pData, &pCtx->ptsList[index], - pCtx->inputType); + do_bottom_function_add(pRes, pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, + NULL, 0); - return true; + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->hasResult = DATA_SET_FLAG; } -static void bottom_dist_merge(SQLFunctionCtx *pCtx) { - STopBotRuntime *pInput = (STopBotRuntime *)GET_INPUT_CHAR(pCtx); +static void bottom_func_merge(SQLFunctionCtx *pCtx) { + char *input = GET_INPUT_CHAR(pCtx); + + STopBotInfo *pInput = (STopBotInfo *)input; if (pInput->num <= 0) { return; } - STopBotRuntime *pRes = (STopBotRuntime *)pCtx->aOutputBuf; + // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary + buildTopBotStruct(pInput, pCtx); + + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pResInfo->superTableQ && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); + + STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); + for (int32_t i = 0; i < pInput->num; ++i) { - bottom_function_do_add(&pRes->num, pCtx->param[0].i64Key, pRes->res, &pInput->res[i].v.i64Key, - &pInput->res[i].timestamp, pCtx->inputType); + do_bottom_function_add(pOutput, pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, + pCtx->inputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } - - pCtx->numOfIteratedElems = pRes->num; } -static void bottom_dist_second_merge(SQLFunctionCtx *pCtx) { - STopBotRuntime *pInput = (STopBotRuntime *)GET_INPUT_CHAR(pCtx); +static void bottom_func_second_merge(SQLFunctionCtx *pCtx) { + STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_CHAR(pCtx); + + // construct the input data struct from binary data + buildTopBotStruct(pInput, pCtx); - /* the intermediate result is binary, we only use the output data type */ + STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); + + // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { - bottom_function_do_add(&pCtx->intermediateBuf[0].i64Key, pCtx->param[0].i64Key, pCtx->intermediateBuf[1].pz, - &pInput->res[i].v.i64Key, &pInput->res[i].timestamp, pCtx->outputType); + do_bottom_function_add(pOutput, pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, + pCtx->outputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } - SET_VAL(pCtx, pInput->num, pCtx->intermediateBuf[0].i64Key); + SET_VAL(pCtx, pInput->num, pOutput->num); + + if (pOutput->num > 0) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->hasResult = DATA_SET_FLAG; + } } -/////////////////////////////////////////////////////////////////////////////////////////////// +static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + + // data in temporary list is less than the required number of results, not enough qualified number of results + STopBotInfo *pRes = pResInfo->interResultBuf; + if (pRes->num == 0) { // no result + assert(pResInfo->hasResult != DATA_SET_FLAG); + // TODO: + } + + GET_RES_INFO(pCtx)->numOfRes = pRes->num; + tValuePair **tvp = pRes->res; + + // user specify the order of output by sort the result according to timestamp + if (pCtx->param[1].i64Key == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + __compar_fn_t comparator = (pCtx->param[2].i64Key == TSQL_SO_ASC) ? resAscComparFn : resDescComparFn; + qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); + } else if (pCtx->param[1].i64Key > PRIMARYKEY_TIMESTAMP_COL_INDEX) { + __compar_fn_t comparator = (pCtx->param[2].i64Key == TSQL_SO_ASC) ? resDataAscComparFn : resDataDescComparFn; + qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); + } + + GET_TRUE_DATA_TYPE(); + copyTopBotRes(pCtx, type); -static void percentile_function_setup(SQLFunctionCtx *pCtx) { - pCtx->intermediateBuf[0].nType = TSDB_DATA_TYPE_DOUBLE; + resetResultInfo(pResInfo); +} - const int32_t MAX_AVAILABLE_BUFFER_SIZE = 1 << 20; +/////////////////////////////////////////////////////////////////////////////////////////////// +static bool percentile_function_setup(SQLFunctionCtx *pCtx) { + const int32_t MAX_AVAILABLE_BUFFER_SIZE = 1 << 20; // 1MB const int32_t NUMOFCOLS = 1; - if (pCtx->intermediateBuf[2].pz != NULL) { - assert(pCtx->intermediateBuf[1].pz != NULL); - return; + if (!function_setup(pCtx)) { + return false; } - SSchema field[1] = { - {pCtx->inputType, "dummyCol", 0, pCtx->inputBytes}, - }; + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SSchema field[1] = {{pCtx->inputType, "dummyCol", 0, pCtx->inputBytes}}; + tColModel *pModel = tColModelCreate(field, 1, 1000); int32_t orderIdx = 0; // tOrderDesc object - pCtx->intermediateBuf[2].pz = tOrderDesCreate(&orderIdx, NUMOFCOLS, pModel, TSQL_SO_DESC); + tOrderDescriptor *pDesc = tOrderDesCreate(&orderIdx, NUMOFCOLS, pModel, TSQL_SO_DESC); - tMemBucketCreate((tMemBucket **)&(pCtx->intermediateBuf[1].pz), 1024, MAX_AVAILABLE_BUFFER_SIZE, pCtx->inputBytes, - pCtx->inputType, (tOrderDescriptor *)pCtx->intermediateBuf[2].pz); + ((SPercentileInfo *)(pResInfo->interResultBuf))->pMemBucket = + tMemBucketCreate(1024, MAX_AVAILABLE_BUFFER_SIZE, pCtx->inputBytes, pCtx->inputType, pDesc); - INIT_VAL(pCtx); + return true; } -static bool percentile_function(SQLFunctionCtx *pCtx) { +static void percentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - if (pCtx->hasNullValue) { - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); - if (isNull(data, pCtx->inputType)) { - continue; - } + SResultInfo * pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo *pInfo = pResInfo->interResultBuf; - notNullElems += 1; - tMemBucketPut((tMemBucket *)(pCtx->intermediateBuf[1].pz), data, 1); - } - } else { - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); - tMemBucketPut((tMemBucket *)(pCtx->intermediateBuf[1].pz), data, 1); + for (int32_t i = 0; i < pCtx->size; ++i) { + char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + continue; } - notNullElems += pCtx->size; + notNullElems += 1; + tMemBucketPut(pInfo->pMemBucket, data, 1); } SET_VAL(pCtx, notNullElems, 1); - return true; + pResInfo->hasResult = DATA_SET_FLAG; } -static bool percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + + SPercentileInfo *pInfo = (SPercentileInfo *)pResInfo->interResultBuf; + tMemBucketPut(pInfo->pMemBucket, pData, 1); + SET_VAL(pCtx, 1, 1); - tMemBucketPut((tMemBucket *)(pCtx->intermediateBuf[1].pz), pData, 1); - return true; + pResInfo->hasResult = DATA_SET_FLAG; } static void percentile_finalizer(SQLFunctionCtx *pCtx) { double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64Key : pCtx->param[0].dKey; - if (pCtx->numOfIteratedElems > 0) { // check for null - double val = getPercentile(pCtx->intermediateBuf[1].pz, v); - *((double *)pCtx->aOutputBuf) = val; + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + tMemBucket * pMemBucket = ((SPercentileInfo *)pResInfo->interResultBuf)->pMemBucket; + + if (pMemBucket->numOfElems > 0) { // check for null + *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v); } else { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); } - tMemBucketDestroy((tMemBucket **)&(pCtx->intermediateBuf[1].pz)); - tOrderDescDestroy(pCtx->intermediateBuf[2].pz); + tOrderDescDestroy(pMemBucket->pOrderDesc); + tMemBucketDestroy(pMemBucket); - pCtx->intermediateBuf[1].pz = NULL; - pCtx->intermediateBuf[2].pz = NULL; + resetResultInfo(GET_RES_INFO(pCtx)); } -static bool apercentile_function(SQLFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - if (pCtx->hasNullValue) { - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); - if (isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems += 1; - double v = 0; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - v = *(int8_t *)data; - break; - case TSDB_DATA_TYPE_SMALLINT: - v = *(int16_t *)data; - break; - case TSDB_DATA_TYPE_BIGINT: - v = *(int64_t *)data; - break; - case TSDB_DATA_TYPE_FLOAT: - v = *(float *)data; - break; - case TSDB_DATA_TYPE_DOUBLE: - v = *(double *)data; - break; - default: - v = *(int32_t *)data; - break; - } +////////////////////////////////////////////////////////////////////////////////// +static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); - tHistogramAdd(&pCtx->param[1].pz, v); - } + if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { + return pCtx->aOutputBuf; } else { - for (int32_t i = 0; i < pCtx->size; ++i) { - char * data = GET_INPUT_CHAR_INDEX(pCtx, i); - double v = 0; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - v = *(int8_t *)data; - break; - case TSDB_DATA_TYPE_SMALLINT: - v = *(int16_t *)data; - break; - case TSDB_DATA_TYPE_BIGINT: - v = *(int64_t *)data; - break; - case TSDB_DATA_TYPE_FLOAT: - v = *(float *)data; - break; - case TSDB_DATA_TYPE_DOUBLE: - v = *(double *)data; - break; - default: - v = *(int32_t *)data; - break; - } - - tHistogramAdd(&pCtx->param[1].pz, v); - } - - notNullElems += pCtx->size; + return pResInfo->interResultBuf; } - - SET_VAL(pCtx, notNullElems, 1); - return true; } -static bool apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; +static bool apercentile_function_setup(SQLFunctionCtx *pCtx) { + if (!function_setup(pCtx)) { + return false; } - double v = 0; - SET_VAL(pCtx, 1, 1); - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - v = *(int8_t *)pData; - break; - case TSDB_DATA_TYPE_SMALLINT: - v = *(int16_t *)pData; - break; - case TSDB_DATA_TYPE_BIGINT: - v = *(int64_t *)pData; - break; - case TSDB_DATA_TYPE_FLOAT: - v = *(float *)pData; - break; - case TSDB_DATA_TYPE_DOUBLE: - v = *(double *)pData; - break; - default: - v = *(int32_t *)pData; - break; - } + SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - tHistogramAdd(&pCtx->param[1].pz, v); + char *tmp = (char *)pInfo + sizeof(SAPercentileInfo); + pInfo->pHisto = tHistogramCreateFrom(tmp, MAX_HISTOGRAM_BIN); return true; } -static void apercentile_finalizer(SQLFunctionCtx *pCtx) { - double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64Key : pCtx->param[0].dKey; - - if (pCtx->numOfIteratedElems > 0) { // check for null - double ratio[] = {v}; - double *res = tHistogramUniform(pCtx->param[1].pz, ratio, 1); - memcpy(pCtx->aOutputBuf, res, sizeof(double)); - free(res); - } else { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); - } - - SET_VAL(pCtx, pCtx->numOfIteratedElems, 1); - tHistogramDestroy(&(pCtx->param[1].pz)); -} - -static void apercentile_dist_function_setup(SQLFunctionCtx *pCtx) { - function_setup(pCtx); - if (pCtx->outputType == TSDB_DATA_TYPE_BINARY) { - tHistogramCreateFrom(pCtx->aOutputBuf, MAX_HISTOGRAM_BIN); - } else { /* for secondary merge at client-side */ - pCtx->param[1].pz = tHistogramCreate(MAX_HISTOGRAM_BIN); - } -} - -static bool apercentile_dist_intern_function(SQLFunctionCtx *pCtx) { +static void apercentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SHistogramInfo *pHisto = (SHistogramInfo *)pCtx->aOutputBuf; - - if (pCtx->hasNullValue) { - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); - if (isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems += 1; - double v = 0; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - v = *(int8_t *)data; - break; - case TSDB_DATA_TYPE_SMALLINT: - v = *(int16_t *)data; - break; - case TSDB_DATA_TYPE_BIGINT: - v = *(int64_t *)data; - break; - case TSDB_DATA_TYPE_FLOAT: - v = *(float *)data; - break; - case TSDB_DATA_TYPE_DOUBLE: - v = *(double *)data; - break; - default: - v = *(int32_t *)data; - break; - } + SResultInfo * pResInfo = GET_RES_INFO(pCtx); + SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - tHistogramAdd(&pHisto, v); + for (int32_t i = 0; i < pCtx->size; ++i) { + char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + continue; } - } else { - for (int32_t i = 0; i < pCtx->size; ++i) { - char * data = GET_INPUT_CHAR_INDEX(pCtx, i); - double v = 0; - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - v = *(int8_t *)data; - break; - case TSDB_DATA_TYPE_SMALLINT: - v = *(int16_t *)data; - break; - case TSDB_DATA_TYPE_BIGINT: - v = *(int64_t *)data; - break; - case TSDB_DATA_TYPE_FLOAT: - v = *(float *)data; - break; - case TSDB_DATA_TYPE_DOUBLE: - v = *(double *)data; - break; - default: - v = *(int32_t *)data; - break; - } + notNullElems += 1; + double v = 0; - tHistogramAdd(&pHisto, v); + switch (pCtx->inputType) { + case TSDB_DATA_TYPE_TINYINT: + v = GET_INT8_VAL(data); + break; + case TSDB_DATA_TYPE_SMALLINT: + v = GET_INT16_VAL(data); + break; + case TSDB_DATA_TYPE_BIGINT: + v = GET_INT64_VAL(data); + break; + case TSDB_DATA_TYPE_FLOAT: + v = GET_FLOAT_VAL(data); + break; + case TSDB_DATA_TYPE_DOUBLE: + v = GET_DOUBLE_VAL(data); + break; + default: + v = GET_INT32_VAL(data); + break; } - notNullElems += pCtx->size; + tHistogramAdd(&pInfo->pHisto, v); + } + + if (!pCtx->hasNull) { + assert(pCtx->size == notNullElems); } SET_VAL(pCtx, notNullElems, 1); - return true; + + if (notNullElems > 0) { + pResInfo->hasResult = DATA_SET_FLAG; + } } -static bool apercentile_dist_intern_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } - SHistogramInfo *pHisto = (SHistogramInfo *)pCtx->aOutputBuf; - SET_VAL(pCtx, 1, 1); + SResultInfo * pResInfo = GET_RES_INFO(pCtx); + SAPercentileInfo *pInfo = getAPerctInfo(pCtx); // pResInfo->interResultBuf; double v = 0; switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: - v = *(int8_t *)pData; + v = GET_INT8_VAL(pData); break; case TSDB_DATA_TYPE_SMALLINT: - v = *(int16_t *)pData; + v = GET_INT16_VAL(pData); break; case TSDB_DATA_TYPE_BIGINT: - v = *(int64_t *)pData; + v = GET_INT64_VAL(pData); break; case TSDB_DATA_TYPE_FLOAT: - v = *(float *)pData; + v = GET_FLOAT_VAL(pData); break; case TSDB_DATA_TYPE_DOUBLE: - v = *(double *)pData; + v = GET_DOUBLE_VAL(pData); break; default: - v = *(int32_t *)pData; + v = GET_INT32_VAL(pData); break; } - tHistogramAdd(&pHisto, v); - return true; + tHistogramAdd(&pInfo->pHisto, v); + + SET_VAL(pCtx, 1, 1); + pResInfo->hasResult = DATA_SET_FLAG; } -static void apercentile_dist_merge(SQLFunctionCtx *pCtx) { - SHistogramInfo *pInput = (SHistogramInfo *)GET_INPUT_CHAR(pCtx); - if (pInput->numOfElems <= 0) { +static void apercentile_func_merge(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pResInfo->superTableQ); + + SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); + + pInput->pHisto = (char *)pInput + sizeof(SAPercentileInfo); + pInput->pHisto->elems = (char *)pInput->pHisto + sizeof(SHistogramInfo); + + if (pInput->pHisto->numOfElems <= 0) { return; } size_t size = sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1); - SHistogramInfo *pHisto = (SHistogramInfo *)pCtx->aOutputBuf; + SAPercentileInfo *pOutput = getAPerctInfo(pCtx); //(SAPercentileInfo *)pCtx->aOutputBuf; + SHistogramInfo * pHisto = pOutput->pHisto; + if (pHisto->numOfElems <= 0) { - char *ptr = pHisto->elems; - memcpy(pHisto, pInput, size); - pHisto->elems = ptr; + memcpy(pHisto, pInput->pHisto, size); + pHisto->elems = (char *)pHisto + sizeof(SHistogramInfo); } else { - pInput->elems = (char *)pInput + sizeof(SHistogramInfo); pHisto->elems = (char *)pHisto + sizeof(SHistogramInfo); - SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput, MAX_HISTOGRAM_BIN); - memcpy(pHisto, pRes, size); - + SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); + memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); pHisto->elems = (char *)pHisto + sizeof(SHistogramInfo); + tHistogramDestroy(&pRes); } - pCtx->numOfIteratedElems += 1; + SET_VAL(pCtx, 1, 1); + pResInfo->hasResult = DATA_SET_FLAG; } -static void apercentile_dist_second_merge(SQLFunctionCtx *pCtx) { - SHistogramInfo *pInput = (SHistogramInfo *)GET_INPUT_CHAR(pCtx); - if (pInput->numOfElems <= 0) { +static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { + SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); + + pInput->pHisto = (char *)pInput + sizeof(SAPercentileInfo); + pInput->pHisto->elems = (char *)pInput->pHisto + sizeof(SHistogramInfo); + + if (pInput->pHisto->numOfElems <= 0) { return; } - SHistogramInfo *pHisto = (SHistogramInfo *)pCtx->param[1].pz; + SAPercentileInfo *pOutput = getAPerctInfo(pCtx); + SHistogramInfo * pHisto = pOutput->pHisto; + if (pHisto->numOfElems <= 0) { - memcpy(pHisto, pInput, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); + memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); pHisto->elems = (char *)pHisto + sizeof(SHistogramInfo); } else { - pInput->elems = (char *)pInput + sizeof(SHistogramInfo); pHisto->elems = (char *)pHisto + sizeof(SHistogramInfo); - SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput, MAX_HISTOGRAM_BIN); - tHistogramDestroy(&pCtx->param[1].pz); - pCtx->param[1].pz = pRes; + SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); + tHistogramDestroy(&pOutput->pHisto); + pOutput->pHisto = pRes; + } + + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->hasResult = DATA_SET_FLAG; + SET_VAL(pCtx, 1, 1); +} + +static void apercentile_finalizer(SQLFunctionCtx *pCtx) { + double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i64Key : pCtx->param[0].dKey; + + SResultInfo * pResInfo = GET_RES_INFO(pCtx); + SAPercentileInfo *pOutput = pResInfo->interResultBuf; + + if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null + assert(pOutput->pHisto->numOfElems > 0); + + double ratio[] = {v}; + double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); + + memcpy(pCtx->aOutputBuf, res, sizeof(double)); + free(res); + } else { + setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + return; + } + } else { + if (pOutput->pHisto->numOfElems > 0) { + double ratio[] = {v}; + + double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); + memcpy(pCtx->aOutputBuf, res, sizeof(double)); + free(res); + } else { // no need to free + setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + return; + } } - pCtx->numOfIteratedElems += 1; + resetResultInfo(pResInfo); } -static void leastsquares_function_setup(SQLFunctionCtx *pCtx) { - if (pCtx->intermediateBuf[1].pz != NULL) { - return; +///////////////////////////////////////////////////////////////////////////////// +static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) { + if (!function_setup(pCtx)) { + return false; } - // 2*3 matrix - INIT_VAL(pCtx); - pCtx->intermediateBuf[1].pz = (double(*)[3])calloc(1, sizeof(double) * 2 * 3); - // set the start x-axle value - pCtx->intermediateBuf[0].dKey = pCtx->param[0].dKey; + SResultInfo * pResInfo = GET_RES_INFO(pCtx); + SLeastsquareInfo *pInfo = pResInfo->interResultBuf; + + // 2*3 matrix + pInfo->startVal = pCtx->param[0].dKey; + return true; } #define LEASTSQR_CAL(p, x, y, index, step) \ @@ -2646,19 +2674,22 @@ static void leastsquares_function_setup(SQLFunctionCtx *pCtx) { } while (0) #define LEASTSQR_CAL_LOOP(ctx, param, x, y, tsdbType, n, step) \ - for (int32_t i = 0; i < ctx->size; ++i) { \ - if (isNull((char *)&(y)[i], tsdbType)) { \ + for (int32_t i = 0; i < (ctx)->size; ++i) { \ + if ((ctx)->hasNull && isNull((char *)&(y)[i], tsdbType)) { \ continue; \ } \ - n++; \ + (n)++; \ LEASTSQR_CAL(param, x, y, i, step); \ } -static bool leastsquares_function(SQLFunctionCtx *pCtx) { - double(*param)[3] = (double(*)[3])pCtx->intermediateBuf[1].pz; +static void leastsquares_function(SQLFunctionCtx *pCtx) { + SResultInfo * pResInfo = GET_RES_INFO(pCtx); + SLeastsquareInfo *pInfo = pResInfo->interResultBuf; + + double(*param)[3] = pInfo->mat; + double x = pInfo->startVal; - double x = pCtx->intermediateBuf[0].dKey; - void * pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_CHAR(pCtx); int32_t numOfElem = 0; switch (pCtx->inputType) { @@ -2666,7 +2697,7 @@ static bool leastsquares_function(SQLFunctionCtx *pCtx) { int32_t *p = pData; // LEASTSQR_CAL_LOOP(pCtx, param, pParamData, p); for (int32_t i = 0; i < pCtx->size; ++i) { - if (isNull(p, pCtx->inputType)) { + if (pCtx->hasNull && isNull(p, pCtx->inputType)) { continue; } @@ -2707,70 +2738,83 @@ static bool leastsquares_function(SQLFunctionCtx *pCtx) { }; } - pCtx->intermediateBuf[0].dKey = x; + pInfo->startVal = x; + pInfo->num += numOfElem; + + if (pInfo->num > 0) { + pResInfo->hasResult = DATA_SET_FLAG; + } SET_VAL(pCtx, numOfElem, 1); - return true; } -static bool leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } - SET_VAL(pCtx, 1, 1); - double(*param)[3] = (double(*)[3])pCtx->intermediateBuf[1].pz; + SResultInfo * pResInfo = GET_RES_INFO(pCtx); + SLeastsquareInfo *pInfo = pResInfo->interResultBuf; + + double(*param)[3] = pInfo->mat; switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { int32_t *p = pData; - LEASTSQR_CAL(param, pCtx->intermediateBuf[0].dKey, p, index, pCtx->param[1].dKey); + LEASTSQR_CAL(param, pInfo->startVal, p, index, pCtx->param[1].dKey); break; }; case TSDB_DATA_TYPE_TINYINT: { int8_t *p = pData; - LEASTSQR_CAL(param, pCtx->intermediateBuf[0].dKey, p, index, pCtx->param[1].dKey); + LEASTSQR_CAL(param, pInfo->startVal, p, index, pCtx->param[1].dKey); break; } case TSDB_DATA_TYPE_SMALLINT: { int16_t *p = pData; - LEASTSQR_CAL(param, pCtx->intermediateBuf[0].dKey, p, index, pCtx->param[1].dKey); + LEASTSQR_CAL(param, pInfo->startVal, p, index, pCtx->param[1].dKey); break; } case TSDB_DATA_TYPE_BIGINT: { int64_t *p = pData; - LEASTSQR_CAL(param, pCtx->intermediateBuf[0].dKey, p, index, pCtx->param[1].dKey); + LEASTSQR_CAL(param, pInfo->startVal, p, index, pCtx->param[1].dKey); break; } case TSDB_DATA_TYPE_FLOAT: { float *p = pData; - LEASTSQR_CAL(param, pCtx->intermediateBuf[0].dKey, p, index, pCtx->param[1].dKey); + LEASTSQR_CAL(param, pInfo->startVal, p, index, pCtx->param[1].dKey); break; } case TSDB_DATA_TYPE_DOUBLE: { double *p = pData; - LEASTSQR_CAL(param, pCtx->intermediateBuf[0].dKey, p, index, pCtx->param[1].dKey); + LEASTSQR_CAL(param, pInfo->startVal, p, index, pCtx->param[1].dKey); break; } default: - pError("error data type in leastsquares function:%d", pCtx->inputType); + pError("error data type in leastsquare function:%d", pCtx->inputType); }; - return true; + SET_VAL(pCtx, 1, 1); + pInfo->num += 1; + + if (pInfo->num > 0) { + pResInfo->hasResult = DATA_SET_FLAG; + } } -static void leastsquare_finalizer(SQLFunctionCtx *pCtx) { - /* no data in query */ - if (pCtx->numOfIteratedElems <= 0) { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); - tfree(pCtx->intermediateBuf[1].pz); +static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { + // no data in query + SResultInfo * pResInfo = GET_RES_INFO(pCtx); + SLeastsquareInfo *pInfo = pResInfo->interResultBuf; + if (pInfo->num == 0) { + setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; } - double(*param)[3] = (double(*)[3])pCtx->intermediateBuf[1].pz; - param[1][1] = pCtx->numOfIteratedElems; + double(*param)[3] = pInfo->mat; + + param[1][1] = pInfo->num; param[1][0] = param[0][1]; param[0][0] -= param[1][0] * (param[0][1] / param[1][1]); @@ -2783,21 +2827,20 @@ static void leastsquare_finalizer(SQLFunctionCtx *pCtx) { param[1][2] /= param[1][1]; sprintf(pCtx->aOutputBuf, "(%lf, %lf)", param[0][2], param[1][2]); - tfree(pCtx->intermediateBuf[1].pz); + resetResultInfo(GET_RES_INFO(pCtx)); } -static bool date_col_output_function(SQLFunctionCtx *pCtx) { +static void date_col_output_function(SQLFunctionCtx *pCtx) { if (pCtx->scanFlag == SUPPLEMENTARY_SCAN) { - return true; + return; } SET_VAL(pCtx, pCtx->size, 1); *(int64_t *)(pCtx->aOutputBuf) = pCtx->nStartQueryTimestamp; - return true; } -static bool col_project_function(SQLFunctionCtx *pCtx) { - INC_INIT_VAL(pCtx, pCtx->size, pCtx->size); +static void col_project_function(SQLFunctionCtx *pCtx) { + INC_INIT_VAL(pCtx, pCtx->size); char *pDest = 0; if (pCtx->order == TSQL_SO_ASC) { @@ -2810,17 +2853,21 @@ static bool col_project_function(SQLFunctionCtx *pCtx) { memcpy(pDest, pData, (size_t)pCtx->size * pCtx->inputBytes); pCtx->aOutputBuf += pCtx->size * pCtx->outputBytes * GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - return true; } -static bool col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { - INC_INIT_VAL(pCtx, 1, 1); +static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + + // only one output + if (pCtx->param[0].i64Key == 1 && pResInfo->numOfRes >= 1) { + return; + } + INC_INIT_VAL(pCtx, 1); char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); pCtx->aOutputBuf += pCtx->inputBytes * GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - return true; } /** @@ -2828,70 +2875,66 @@ static bool col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { * @param pCtx * @return */ -static bool tag_project_function(SQLFunctionCtx *pCtx) { - INC_INIT_VAL(pCtx, pCtx->size, pCtx->size); +static void tag_project_function(SQLFunctionCtx *pCtx) { + INC_INIT_VAL(pCtx, pCtx->size); assert(pCtx->inputBytes == pCtx->outputBytes); int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); for (int32_t i = 0; i < pCtx->size; ++i) { - tVariantDump(&pCtx->intermediateBuf[3], pCtx->aOutputBuf, pCtx->intermediateBuf[3].nType); + tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType); pCtx->aOutputBuf += pCtx->outputBytes * factor; } - return true; } -static bool tag_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { - INC_INIT_VAL(pCtx, 1, 1); - tVariantDump(&pCtx->intermediateBuf[3], pCtx->aOutputBuf, pCtx->intermediateBuf[3].nType); +static void tag_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { + INC_INIT_VAL(pCtx, 1); + tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->tag.nType); pCtx->aOutputBuf += pCtx->outputBytes * GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - return true; } /** * used in group by clause. when applying group by tags, the tags value is - * assign - * by using tag function. + * assign by using tag function. * NOTE: there is only ONE output for ONE query range * @param pCtx * @return */ -static bool tag_function(SQLFunctionCtx *pCtx) { +static void tag_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, 1, 1); - tVariantDump(&pCtx->intermediateBuf[3], pCtx->aOutputBuf, pCtx->intermediateBuf[3].nType); - return true; + tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->tag.nType); } -static bool tag_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void tag_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); - tVariantDump(&pCtx->intermediateBuf[3], pCtx->aOutputBuf, pCtx->intermediateBuf[3].nType); - return true; + tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->tag.nType); } -static bool copy_function(SQLFunctionCtx *pCtx) { +static void copy_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); char *pData = GET_INPUT_CHAR(pCtx); assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); - return false; } enum { INITIAL_VALUE_NOT_ASSIGNED = 0, }; -static void diff_function_setup(SQLFunctionCtx *pCtx) { - function_setup(pCtx); - pCtx->intermediateBuf[1].nType = INITIAL_VALUE_NOT_ASSIGNED; +static bool diff_function_setup(SQLFunctionCtx *pCtx) { + if (function_setup(pCtx)) { + return false; + } + // diff function require the value is set to -1 + pCtx->param[1].nType = INITIAL_VALUE_NOT_ASSIGNED; + return false; } // TODO difference in date column -static bool diff_function(SQLFunctionCtx *pCtx) { - pCtx->numOfIteratedElems += pCtx->size; - - void *pData = GET_INPUT_CHAR(pCtx); - bool isFirstBlock = (pCtx->intermediateBuf[1].nType == INITIAL_VALUE_NOT_ASSIGNED); +static void diff_function(SQLFunctionCtx *pCtx) { + void *data = GET_INPUT_CHAR(pCtx); + bool isFirstBlock = (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED); int32_t notNullElems = 0; @@ -2902,184 +2945,186 @@ static bool diff_function(SQLFunctionCtx *pCtx) { switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { - int32_t *pDd = (int32_t *)pData; + int32_t *pData = (int32_t *)data; int32_t *pOutput = (int32_t *)pCtx->aOutputBuf; for (; i < pCtx->size && i >= 0; i += step) { - if (isNull(&pDd[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { continue; } - if (pCtx->intermediateBuf[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - pCtx->intermediateBuf[1].i64Key = pDd[i]; - pCtx->intermediateBuf[1].nType = pCtx->inputType; + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + pCtx->param[1].i64Key = pData[i]; + pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { - *pOutput = pDd[i] - pCtx->intermediateBuf[1].i64Key; + *pOutput = pData[i] - pCtx->param[1].i64Key; *pTimestamp = pCtx->ptsList[i]; + pOutput += step; pTimestamp += step; } else { - *pOutput = pDd[i] - pDd[i - step]; + *pOutput = pData[i] - pData[i - step]; *pTimestamp = pCtx->ptsList[i]; + pOutput += step; pTimestamp += step; } - pCtx->intermediateBuf[1].i64Key = pDd[i]; - pCtx->intermediateBuf[1].nType = pCtx->inputType; + pCtx->param[1].i64Key = pData[i]; + pCtx->param[1].nType = pCtx->inputType; notNullElems++; } break; }; case TSDB_DATA_TYPE_BIGINT: { - int64_t *pDd = (int64_t *)pData; + int64_t *pData = (int64_t *)data; int64_t *pOutput = (int64_t *)pCtx->aOutputBuf; for (; i < pCtx->size && i >= 0; i += step) { - if (isNull(&pDd[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { continue; } - if (pCtx->intermediateBuf[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - pCtx->intermediateBuf[1].i64Key = pDd[i]; - pCtx->intermediateBuf[1].nType = pCtx->inputType; + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + pCtx->param[1].i64Key = pData[i]; + pCtx->param[1].nType = pCtx->inputType; } else if (i == 0) { - *pOutput = pDd[i] - pCtx->intermediateBuf[1].i64Key; + *pOutput = pData[i] - pCtx->param[1].i64Key; *pTimestamp = pCtx->ptsList[i]; pOutput += step; pTimestamp += step; } else { - *pOutput = pDd[i] - pDd[i - 1]; + *pOutput = pData[i] - pData[i - 1]; *pTimestamp = pCtx->ptsList[i]; pOutput += step; pTimestamp += step; } - pCtx->intermediateBuf[1].i64Key = pDd[i]; - pCtx->intermediateBuf[1].nType = pCtx->inputType; + pCtx->param[1].i64Key = pData[i]; + pCtx->param[1].nType = pCtx->inputType; notNullElems++; } break; }; case TSDB_DATA_TYPE_DOUBLE: { - double *pDd = (double *)pData; + double *pData = (double *)data; double *pOutput = (double *)pCtx->aOutputBuf; for (; i < pCtx->size && i >= 0; i += step) { - if (isNull(&pDd[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { continue; } - if (pCtx->intermediateBuf[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - pCtx->intermediateBuf[1].dKey = pDd[i]; - pCtx->intermediateBuf[1].nType = pCtx->inputType; + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + pCtx->param[1].dKey = pData[i]; + pCtx->param[1].nType = pCtx->inputType; } else if (i == 0) { - *pOutput = pDd[i] - pCtx->intermediateBuf[1].dKey; + *pOutput = pData[i] - pCtx->param[1].dKey; *pTimestamp = pCtx->ptsList[i]; pOutput += step; pTimestamp += step; } else { - *pOutput = pDd[i] - pDd[i - 1]; + *pOutput = pData[i] - pData[i - 1]; *pTimestamp = pCtx->ptsList[i]; pOutput += step; pTimestamp += step; } - memcpy(&pCtx->intermediateBuf[1].i64Key, &pDd[i], pCtx->inputBytes); - pCtx->intermediateBuf[1].nType = pCtx->inputType; + pCtx->param[1].dKey = pData[i]; + pCtx->param[1].nType = pCtx->inputType; notNullElems++; } break; }; case TSDB_DATA_TYPE_FLOAT: { - float *pDd = (float *)pData; + float *pData = (float *)data; float *pOutput = (float *)pCtx->aOutputBuf; for (; i < pCtx->size && i >= 0; i += step) { - if (isNull(&pDd[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { continue; } - if (pCtx->intermediateBuf[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - pCtx->intermediateBuf[1].dKey = pDd[i]; - pCtx->intermediateBuf[1].nType = pCtx->inputType; + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + pCtx->param[1].dKey = pData[i]; + pCtx->param[1].nType = pCtx->inputType; } else if (i == 0) { - *pOutput = pDd[i] - pCtx->intermediateBuf[1].dKey; + *pOutput = pData[i] - pCtx->param[1].dKey; *pTimestamp = pCtx->ptsList[i]; pOutput += step; pTimestamp += step; } else { - *pOutput = pDd[i] - pDd[i - 1]; + *pOutput = pData[i] - pData[i - 1]; *pTimestamp = pCtx->ptsList[i]; pOutput += step; pTimestamp += step; } // keep the last value, the remain may be all null - pCtx->intermediateBuf[1].dKey = pDd[i]; - pCtx->intermediateBuf[1].nType = pCtx->inputType; + pCtx->param[1].dKey = pData[i]; + pCtx->param[1].nType = pCtx->inputType; notNullElems++; } break; }; case TSDB_DATA_TYPE_SMALLINT: { - int16_t *pDd = (int16_t *)pData; + int16_t *pData = (int16_t *)data; int16_t *pOutput = (int16_t *)pCtx->aOutputBuf; for (; i < pCtx->size && i >= 0; i += step) { - if (isNull(&pDd[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { continue; } - if (pCtx->intermediateBuf[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - pCtx->intermediateBuf[1].i64Key = pDd[i]; - pCtx->intermediateBuf[1].nType = pCtx->inputType; + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + pCtx->param[1].i64Key = pData[i]; + pCtx->param[1].nType = pCtx->inputType; } else if (i == 0) { - *pOutput = pDd[i] - pCtx->intermediateBuf[1].i64Key; + *pOutput = pData[i] - pCtx->param[1].i64Key; *pTimestamp = pCtx->ptsList[i]; pOutput += step; pTimestamp += step; } else { - *pOutput = pDd[i] - pDd[i - 1]; + *pOutput = pData[i] - pData[i - 1]; *pTimestamp = pCtx->ptsList[i]; pOutput += step; pTimestamp += step; } - pCtx->intermediateBuf[1].i64Key = pDd[i]; - pCtx->intermediateBuf[1].nType = pCtx->inputType; + pCtx->param[1].i64Key = pData[i]; + pCtx->param[1].nType = pCtx->inputType; notNullElems++; } break; }; case TSDB_DATA_TYPE_TINYINT: { - int8_t *pDd = (int8_t *)pData; + int8_t *pData = (int8_t *)data; int8_t *pOutput = (int8_t *)pCtx->aOutputBuf; for (; i < pCtx->size && i >= 0; i += step) { - if (isNull((char *)&pDd[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) { continue; } - if (pCtx->intermediateBuf[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - pCtx->intermediateBuf[1].i64Key = pDd[i]; - pCtx->intermediateBuf[1].nType = pCtx->inputType; + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + pCtx->param[1].i64Key = pData[i]; + pCtx->param[1].nType = pCtx->inputType; } else if (i == 0) { - *pOutput = pDd[i] - pCtx->intermediateBuf[1].i64Key; + *pOutput = pData[i] - pCtx->param[1].i64Key; *pTimestamp = pCtx->ptsList[i]; pOutput += step; pTimestamp += step; } else { - *pOutput = pDd[i] - pDd[i - 1]; + *pOutput = pData[i] - pData[i - 1]; *pTimestamp = pCtx->ptsList[i]; pOutput += step; pTimestamp += step; } - pCtx->intermediateBuf[1].i64Key = pDd[i]; - pCtx->intermediateBuf[1].nType = pCtx->inputType; + pCtx->param[1].i64Key = pData[i]; + pCtx->param[1].nType = pCtx->inputType; notNullElems++; } break; @@ -3088,95 +3133,88 @@ static bool diff_function(SQLFunctionCtx *pCtx) { pError("error input type"); } - if (pCtx->intermediateBuf[1].nType == INITIAL_VALUE_NOT_ASSIGNED || - notNullElems <= 0) { // initial value is not set yet - /* - * 1. current block and blocks before are full of null - * 2. current block may be null value - */ - assert(pCtx->hasNullValue); + // initial value is not set yet + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED || notNullElems <= 0) { + /* + * 1. current block and blocks before are full of null + * 2. current block may be null value + */ + assert(pCtx->hasNull); } else { - if (isFirstBlock) { - pCtx->numOfOutputElems = notNullElems - 1; - } else { - pCtx->numOfOutputElems += notNullElems; - } - int32_t forwardStep = (isFirstBlock) ? notNullElems - 1 : notNullElems; + GET_RES_INFO(pCtx)->numOfRes += forwardStep; + pCtx->aOutputBuf = pCtx->aOutputBuf + forwardStep * pCtx->outputBytes * step; pCtx->ptsOutputBuf = (char *)pCtx->ptsOutputBuf + forwardStep * TSDB_KEYSIZE * step; } - - return true; } -#define TYPE_DIFF_IMPL(ctx, d, type) \ - do { \ - if (ctx->intermediateBuf[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { \ - ctx->intermediateBuf[1].nType = ctx->inputType; \ - *(type *)&ctx->intermediateBuf[1].i64Key = *(type *)d; \ - } else { \ - *(type *)ctx->aOutputBuf = *(type *)d - (*(type *)(&ctx->intermediateBuf[1].i64Key)); \ - *(type *)(&ctx->intermediateBuf[1].i64Key) = *(type *)d; \ - *(int64_t *)ctx->ptsOutputBuf = *(int64_t *)(ctx->ptsList + (TSDB_KEYSIZE)*index); \ - } \ +#define DIFF_IMPL(ctx, d, type) \ + do { \ + if ((ctx)->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { \ + (ctx)->param[1].nType = (ctx)->inputType; \ + *(type *)&(ctx)->param[1].i64Key = *(type *)(d); \ + } else { \ + *(type *)(ctx)->aOutputBuf = *(type *)(d) - (*(type *)(&(ctx)->param[1].i64Key)); \ + *(type *)(&(ctx)->param[1].i64Key) = *(type *)(d); \ + *(int64_t *)(ctx)->ptsOutputBuf = *(int64_t *)((ctx)->ptsList + (TSDB_KEYSIZE)*index); \ + } \ } while (0); -static bool diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } - pCtx->numOfIteratedElems += 1; - if (pCtx->intermediateBuf[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is set - pCtx->numOfOutputElems += 1; + // the output start from the second source element + if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is set + GET_RES_INFO(pCtx)->numOfRes += 1; } int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { - if (pCtx->intermediateBuf[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - pCtx->intermediateBuf[1].nType = pCtx->inputType; - pCtx->intermediateBuf[1].i64Key = *(int32_t *)pData; + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + pCtx->param[1].nType = pCtx->inputType; + pCtx->param[1].i64Key = *(int32_t *)pData; } else { - *(int32_t *)pCtx->aOutputBuf = *(int32_t *)pData - pCtx->intermediateBuf[1].i64Key; - pCtx->intermediateBuf[1].i64Key = *(int32_t *)pData; + *(int32_t *)pCtx->aOutputBuf = *(int32_t *)pData - pCtx->param[1].i64Key; + pCtx->param[1].i64Key = *(int32_t *)pData; *(int64_t *)pCtx->ptsOutputBuf = pCtx->ptsList[index]; } break; }; case TSDB_DATA_TYPE_BIGINT: { - TYPE_DIFF_IMPL(pCtx, pData, int64_t); + DIFF_IMPL(pCtx, pData, int64_t); break; }; case TSDB_DATA_TYPE_DOUBLE: { - TYPE_DIFF_IMPL(pCtx, pData, double); + DIFF_IMPL(pCtx, pData, double); break; }; case TSDB_DATA_TYPE_FLOAT: { - TYPE_DIFF_IMPL(pCtx, pData, float); + DIFF_IMPL(pCtx, pData, float); break; }; case TSDB_DATA_TYPE_SMALLINT: { - TYPE_DIFF_IMPL(pCtx, pData, int16_t); + DIFF_IMPL(pCtx, pData, int16_t); break; }; case TSDB_DATA_TYPE_TINYINT: { - TYPE_DIFF_IMPL(pCtx, pData, int8_t); + DIFF_IMPL(pCtx, pData, int8_t); break; }; default: pError("error input type"); } - if (pCtx->numOfOutputElems > 0) { - pCtx->aOutputBuf = pCtx->aOutputBuf + pCtx->outputBytes * step; + if (GET_RES_INFO(pCtx)->numOfRes > 0) { + pCtx->aOutputBuf += pCtx->outputBytes * step; pCtx->ptsOutputBuf = (char *)pCtx->ptsOutputBuf + TSDB_KEYSIZE * step; } - return true; } char *arithmetic_callback_function(void *param, char *name, int32_t colId) { @@ -3196,26 +3234,25 @@ char *arithmetic_callback_function(void *param, char *name, int32_t colId) { return pSupport->data[colIndexInBuf] + pSupport->offset * pSupport->elemSize[colIndexInBuf]; } -bool arithmetic_function(SQLFunctionCtx *pCtx) { - pCtx->numOfOutputElems += pCtx->size; +static void arithmetic_function(SQLFunctionCtx *pCtx) { + GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[0].pz; tSQLBinaryExprCalcTraverse(sas->pExpr->pBinExprInfo.pBinExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, arithmetic_callback_function); - pCtx->aOutputBuf = pCtx->aOutputBuf + pCtx->outputBytes * pCtx->size * GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - return true; + pCtx->aOutputBuf += pCtx->outputBytes * pCtx->size * GET_FORWARD_DIRECTION_FACTOR(pCtx->order); } -bool arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { - INC_INIT_VAL(pCtx, 1, 1); +static bool arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { + INC_INIT_VAL(pCtx, 1); SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[0].pz; sas->offset = index; tSQLBinaryExprCalcTraverse(sas->pExpr->pBinExprInfo.pBinExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, arithmetic_callback_function); - pCtx->aOutputBuf = pCtx->aOutputBuf + pCtx->outputBytes * GET_FORWARD_DIRECTION_FACTOR(pCtx->order); + pCtx->aOutputBuf += pCtx->outputBytes * GET_FORWARD_DIRECTION_FACTOR(pCtx->order); return true; } @@ -3223,7 +3260,7 @@ bool arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { { \ type *inputData = (type *)data; \ for (int32_t i = 0; i < elemCnt; ++i) { \ - if (isNull((char *)&inputData[i], tsdbType)) { \ + if ((ctx)->hasNull && isNull((char *)&inputData[i], tsdbType)) { \ continue; \ } \ if (inputData[i] < minOutput) { \ @@ -3236,69 +3273,67 @@ bool arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { } \ } -#define LIST_MINMAX(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType) \ - { \ - type *inputData = (type *)data; \ - for (int32_t i = 0; i < elemCnt; ++i) { \ - if (inputData[i] < minOutput) { \ - minOutput = inputData[i]; \ - } \ - if (inputData[i] > maxOutput) { \ - maxOutput = inputData[i]; \ - } \ - } \ - } - -void spread_function_setup(SQLFunctionCtx *pCtx) { - if ((pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE) || - (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) || pCtx->inputType == TSDB_DATA_TYPE_BINARY) { - pCtx->intermediateBuf[0].dKey = DBL_MAX; - pCtx->intermediateBuf[3].dKey = -DBL_MAX; +///////////////////////////////////////////////////////////////////////////////// +static bool spread_function_setup(SQLFunctionCtx *pCtx) { + if (!function_setup(pCtx)) { + return false; + } + + SSpreadInfo *pInfo = GET_RES_INFO(pCtx)->interResultBuf; + + // this is the server-side setup function in client-side, the secondary merge do not need this procedure + if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + pCtx->param[0].dKey = DBL_MAX; + pCtx->param[3].dKey = -DBL_MAX; } else { - pError("illegal data type:%d in spread function query", pCtx->inputType); + pInfo->min = DBL_MAX; + pInfo->max = -DBL_MAX; } - memset(pCtx->aOutputBuf, 0, pCtx->outputBytes); - INIT_VAL(pCtx); + return true; } -bool spread_function(SQLFunctionCtx *pCtx) { +static void spread_function(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SSpreadInfo *pInfo = pResInfo->interResultBuf; + int32_t numOfElems = pCtx->size; - /* column missing cause the hasNullValue to be true */ - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus)) { // Pre-aggregation + // column missing cause the hasNull to be true + if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus)) { if (pCtx->preAggVals.isSet) { - numOfElems = pCtx->size - pCtx->preAggVals.numOfNullPoints; + numOfElems = pCtx->size - pCtx->preAggVals.numOfNull; + + // all data are null in current data block, ignore current data block if (numOfElems == 0) { - /* all data are null in current data block, ignore current data block */ goto _spread_over; } if ((pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)) { - if (pCtx->intermediateBuf[0].dKey > pCtx->preAggVals.min) { - pCtx->intermediateBuf[0].dKey = pCtx->preAggVals.min; + if (pInfo->min > pCtx->preAggVals.min) { + pInfo->min = pCtx->preAggVals.min; } - if (pCtx->intermediateBuf[3].dKey < pCtx->preAggVals.max) { - pCtx->intermediateBuf[3].dKey = pCtx->preAggVals.max; + if (pInfo->max < pCtx->preAggVals.max) { + pInfo->max = pCtx->preAggVals.max; } } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - if (pCtx->intermediateBuf[0].dKey > *(double *)&(pCtx->preAggVals.min)) { - pCtx->intermediateBuf[0].dKey = *(double *)&(pCtx->preAggVals.min); + if (pInfo->min > GET_DOUBLE_VAL(&(pCtx->preAggVals.min))) { + pInfo->min = GET_DOUBLE_VAL(&(pCtx->preAggVals.min)); } - if (pCtx->intermediateBuf[3].dKey < *(double *)&(pCtx->preAggVals.max)) { - pCtx->intermediateBuf[3].dKey = *(double *)&(pCtx->preAggVals.max); + if (pInfo->max < GET_DOUBLE_VAL(&(pCtx->preAggVals.max))) { + pInfo->max = GET_DOUBLE_VAL(&(pCtx->preAggVals.max)); } } } else { - if (pCtx->intermediateBuf[0].dKey > pCtx->intermediateBuf[1].dKey) { - pCtx->intermediateBuf[0].dKey = pCtx->intermediateBuf[1].dKey; + if (pInfo->min > pCtx->param[1].dKey) { + pInfo->min = pCtx->param[1].dKey; } - if (pCtx->intermediateBuf[3].dKey < pCtx->intermediateBuf[2].dKey) { - pCtx->intermediateBuf[3].dKey = pCtx->intermediateBuf[2].dKey; + if (pInfo->max < pCtx->param[2].dKey) { + pInfo->max = pCtx->param[2].dKey; } } @@ -3306,205 +3341,169 @@ bool spread_function(SQLFunctionCtx *pCtx) { } void *pData = GET_INPUT_CHAR(pCtx); + numOfElems = 0; - if (pCtx->hasNullValue) { - numOfElems = 0; + if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { + LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int8_t, pCtx->inputType, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { + LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int16_t, pCtx->inputType, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { + LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int32_t, pCtx->inputType, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT || pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { + LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int64_t, pCtx->inputType, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { + LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, double, pCtx->inputType, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { + LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, float, pCtx->inputType, numOfElems); + } - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_MINMAX_N(pCtx, pCtx->intermediateBuf[0].dKey, pCtx->intermediateBuf[3].dKey, pCtx->size, pData, int8_t, - pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_MINMAX_N(pCtx, pCtx->intermediateBuf[0].dKey, pCtx->intermediateBuf[3].dKey, pCtx->size, pData, int16_t, - pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_MINMAX_N(pCtx, pCtx->intermediateBuf[0].dKey, pCtx->intermediateBuf[3].dKey, pCtx->size, pData, int32_t, - pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT || pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { - LIST_MINMAX_N(pCtx, pCtx->intermediateBuf[0].dKey, pCtx->intermediateBuf[3].dKey, pCtx->size, pData, int64_t, - pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - LIST_MINMAX_N(pCtx, pCtx->intermediateBuf[0].dKey, pCtx->intermediateBuf[3].dKey, pCtx->size, pData, double, - pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - LIST_MINMAX_N(pCtx, pCtx->intermediateBuf[0].dKey, pCtx->intermediateBuf[3].dKey, pCtx->size, pData, float, - pCtx->inputType, numOfElems); - } - } else { - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_MINMAX(pCtx, pCtx->intermediateBuf[0].dKey, pCtx->intermediateBuf[3].dKey, pCtx->size, pData, int8_t, - pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_MINMAX(pCtx, pCtx->intermediateBuf[0].dKey, pCtx->intermediateBuf[3].dKey, pCtx->size, pData, int16_t, - pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_MINMAX(pCtx, pCtx->intermediateBuf[0].dKey, pCtx->intermediateBuf[3].dKey, pCtx->size, pData, int32_t, - pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT || pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { - LIST_MINMAX(pCtx, pCtx->intermediateBuf[0].dKey, pCtx->intermediateBuf[3].dKey, pCtx->size, pData, int64_t, - pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - LIST_MINMAX(pCtx, pCtx->intermediateBuf[0].dKey, pCtx->intermediateBuf[3].dKey, pCtx->size, pData, double, - pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - LIST_MINMAX(pCtx, pCtx->intermediateBuf[0].dKey, pCtx->intermediateBuf[3].dKey, pCtx->size, pData, float, - pCtx->inputType); - } + if (!pCtx->hasNull) { + assert(pCtx->size == numOfElems); } _spread_over: - SET_VAL(pCtx, numOfElems, 1); - return true; + + if (numOfElems > 0) { + pResInfo->hasResult = DATA_SET_FLAG; + pInfo->hasResult = DATA_SET_FLAG; + } + + // keep the data into the final output buffer for super table query since this execution may be the last one + if (pResInfo->superTableQ) { + memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); + } } -bool spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - if (pCtx->hasNullValue && isNull(pData, pCtx->inputType)) { - return true; + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; } SET_VAL(pCtx, 1, 1); + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SSpreadInfo *pInfo = pResInfo->interResultBuf; + double val = 0.0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - val = *(int8_t *)pData; + val = GET_INT8_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - val = *(int16_t *)pData; + val = GET_INT16_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - val = *(int32_t *)pData; - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - val = *(int64_t *)pData; + val = GET_INT32_VAL(pData); + } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT || pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { + val = GET_INT64_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - val = *(double *)pData; + val = GET_DOUBLE_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - val = *(float *)pData; + val = GET_FLOAT_VAL(pData); } - if (pCtx->intermediateBuf[0].dKey > val) { - pCtx->intermediateBuf[0].dKey = val; + // keep the result data in output buffer, not in the intermediate buffer + if (val > pInfo->max) { + pInfo->max = val; } - if (pCtx->intermediateBuf[3].dKey < val) { - pCtx->intermediateBuf[3].dKey = val; + if (val < pInfo->min) { + pInfo->min = val; } - return true; -} + pResInfo->hasResult = DATA_SET_FLAG; + pInfo->hasResult = DATA_SET_FLAG; -void spread_function_finalize(SQLFunctionCtx *pCtx) { - /* - * here we do not check the input data types, because in case of metric query, - * the type of intermediate data is binary - */ - if (pCtx->numOfIteratedElems <= 0) { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); - return; + if (pResInfo->superTableQ) { + memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); } - - *(double *)pCtx->aOutputBuf = pCtx->intermediateBuf[3].dKey - pCtx->intermediateBuf[0].dKey; - SET_VAL(pCtx, pCtx->numOfIteratedElems, 1); } -void spread_dist_function_setup(SQLFunctionCtx *pCtx) { - spread_function_setup(pCtx); +void spread_func_merge(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pResInfo->superTableQ); - /* - * this is the server-side setup function in client-side, the secondary merge do not need this procedure - */ - if (pCtx->outputType == TSDB_DATA_TYPE_BINARY) { - double *pResData = pCtx->aOutputBuf; - pResData[0] = DBL_MAX; // init min value - pResData[1] = -DBL_MAX; // max value - } -} + SSpreadInfo *pResData = pResInfo->interResultBuf; -bool spread_dist_intern_function(SQLFunctionCtx *pCtx) { - // restore value for calculation, since the intermediate result kept in output buffer - SSpreadRuntime *pOutput = (SSpreadRuntime *)pCtx->aOutputBuf; - pCtx->intermediateBuf[0].dKey = pOutput->start; - pCtx->intermediateBuf[3].dKey = pOutput->end; + int32_t notNullElems = 0; + for (int32_t i = 0; i < pCtx->size; ++i) { + SSpreadInfo *input = (SSpreadInfo *)GET_INPUT_CHAR_INDEX(pCtx, i); - spread_function(pCtx); + /* no assign tag, the value is null */ + if (input->hasResult != DATA_SET_FLAG) { + continue; + } - if (pCtx->numOfIteratedElems) { - pOutput->start = pCtx->intermediateBuf[0].dKey; - pOutput->end = pCtx->intermediateBuf[3].dKey; - pOutput->valid = DATA_SET_FLAG; - } + if (pResData->min > input->min) { + pResData->min = input->min; + } - return true; + if (pResData->max < input->max) { + pResData->max = input->max; + } + + pResData->hasResult = DATA_SET_FLAG; + notNullElems++; + } + + if (notNullElems > 0) { + memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); + pResInfo->hasResult = DATA_SET_FLAG; + } } -bool spread_dist_intern_function_f(SQLFunctionCtx *pCtx, int32_t index) { - // restore value for calculation, since the intermediate result kept - // in output buffer - SSpreadRuntime *pOutput = (SSpreadRuntime *)pCtx->aOutputBuf; - pCtx->intermediateBuf[0].dKey = pOutput->start; - pCtx->intermediateBuf[3].dKey = pOutput->end; +/* + * here we set the result value back to the intermediate buffer, to apply the finalize the function + * the final result is generated in spread_function_finalizer + */ +void spread_func_sec_merge(SQLFunctionCtx *pCtx) { + SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_CHAR(pCtx); + if (pData->hasResult != DATA_SET_FLAG) { + return; + } - spread_function_f(pCtx, index); + if (pCtx->param[0].dKey > pData->min) { + pCtx->param[0].dKey = pData->min; + } - /* keep the result data in output buffer, not in the intermediate buffer */ - if (pCtx->numOfIteratedElems) { - pOutput->start = pCtx->intermediateBuf[0].dKey; - pOutput->end = pCtx->intermediateBuf[3].dKey; - pOutput->valid = DATA_SET_FLAG; + if (pCtx->param[3].dKey < pData->max) { + pCtx->param[3].dKey = pData->max; } - return true; + + // pCtx->numOfIteratedElems += 1; + GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } -void spread_dist_merge(SQLFunctionCtx *pCtx) { +void spread_function_finalizer(SQLFunctionCtx *pCtx) { /* - * min,max in double format - * pResData[0] = min - * pResData[1] = max + * here we do not check the input data types, because in case of metric query, + * the type of intermediate data is binary */ - SSpreadRuntime *pResData = (SSpreadRuntime *)pCtx->aOutputBuf; + SResultInfo *pResInfo = GET_RES_INFO(pCtx); - int32_t notNullElems = 0; - for (int32_t i = 0; i < pCtx->size; ++i) { - SSpreadRuntime *input = (SSpreadRuntime *)GET_INPUT_CHAR_INDEX(pCtx, i); + if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - /* no assign tag, the value is null */ - if (input->valid != DATA_SET_FLAG) { - continue; + if (pResInfo->hasResult != DATA_SET_FLAG) { + setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + return; } - if (pResData->start > input->start) { - pResData->start = input->start; - } + *(double *)pCtx->aOutputBuf = pCtx->param[3].dKey - pCtx->param[0].dKey; + } else { + assert((pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE) || + (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); - if (pResData->end < input->end) { - pResData->end = input->end; + SSpreadInfo *pInfo = GET_RES_INFO(pCtx)->interResultBuf; + if (pInfo->hasResult != DATA_SET_FLAG) { + setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + return; } - pResData->valid = DATA_SET_FLAG; - notNullElems++; + *(double *)pCtx->aOutputBuf = pInfo->max - pInfo->min; } - pCtx->numOfIteratedElems += notNullElems; -} - -/* - * here we set the result value back to the intermediate buffer, to apply the finalize the function - * the final result is generated in spread_function_finalize - */ -void spread_dist_second_merge(SQLFunctionCtx *pCtx) { - SSpreadRuntime *pData = (SSpreadRuntime *)GET_INPUT_CHAR(pCtx); - - if (pData->valid != DATA_SET_FLAG) { - return; - } - - if (pCtx->intermediateBuf[0].dKey > pData->start) { - pCtx->intermediateBuf[0].dKey = pData->start; - } - - if (pCtx->intermediateBuf[3].dKey < pData->end) { - pCtx->intermediateBuf[3].dKey = pData->end; - } - - pCtx->numOfIteratedElems += 1; + // SET_VAL(pCtx, pCtx->numOfIteratedElems, 1); + resetResultInfo(GET_RES_INFO(pCtx)); } /* @@ -3557,15 +3556,7 @@ int patternMatch(const char *patterStr, const char *str, size_t size, const SPat c1 = str[j++]; if (j <= size) { - if (c == c1) { - continue; - } - - if (tolower(c) == tolower(c1)) { - continue; - } - - if (c == pInfo->matchOne && c1 != 0) { + if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { continue; } } @@ -3619,14 +3610,7 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c c1 = str[j++]; if (j <= size) { - if (c == c1) { - continue; - } - - if (towlower(c) == towlower(c1)) { - continue; - } - if (c == matchOne && c1 != 0) { + if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) { continue; } } @@ -3634,17 +3618,20 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c return TSDB_PATTERN_NOMATCH; } - return str[j] == 0 ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; + return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } static void getStatics_i8(int64_t *primaryKey, int32_t type, int8_t *data, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int64_t *wsum, int32_t *numOfNull) { + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { *min = INT64_MAX; *max = INT64_MIN; - *wsum = 0; + *minIndex = 0; + *maxIndex = 0; - int64_t lastKey = 0; - int8_t lastVal = TSDB_DATA_TINYINT_NULL; + assert(numOfRow <= INT16_MAX); + + // int64_t lastKey = 0; + // int8_t lastVal = TSDB_DATA_TINYINT_NULL; for (int32_t i = 0; i < numOfRow; ++i) { if (isNull((char *)&data[i], type)) { @@ -3655,34 +3642,38 @@ static void getStatics_i8(int64_t *primaryKey, int32_t type, int8_t *data, int32 *sum += data[i]; if (*min > data[i]) { *min = data[i]; + *minIndex = i; } if (*max < data[i]) { *max = data[i]; + *maxIndex = i; } - if (type != TSDB_DATA_TYPE_BOOL) { - // ignore the bool data type pre-calculation - if (isNull((char *)&lastVal, type)) { - lastKey = primaryKey[i]; - lastVal = data[i]; - } else { - *wsum = lastVal * (primaryKey[i] - lastKey); - lastKey = primaryKey[i]; - lastVal = data[i]; - } - } + // if (type != TSDB_DATA_TYPE_BOOL) { // ignore the bool data type pre-calculation + // if (isNull((char *)&lastVal, type)) { + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } else { + // *wsum = lastVal * (primaryKey[i] - lastKey); + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } + // } } } static void getStatics_i16(int64_t *primaryKey, int16_t *data, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int64_t *wsum, int32_t *numOfNull) { + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { *min = INT64_MAX; *max = INT64_MIN; - *wsum = 0; + *minIndex = 0; + *maxIndex = 0; - int64_t lastKey = 0; - int16_t lastVal = TSDB_DATA_SMALLINT_NULL; + assert(numOfRow <= INT16_MAX); + + // int64_t lastKey = 0; + // int16_t lastVal = TSDB_DATA_SMALLINT_NULL; for (int32_t i = 0; i < numOfRow; ++i) { if (isNull(&data[i], TSDB_DATA_TYPE_SMALLINT)) { @@ -3693,31 +3684,36 @@ static void getStatics_i16(int64_t *primaryKey, int16_t *data, int32_t numOfRow, *sum += data[i]; if (*min > data[i]) { *min = data[i]; + *minIndex = i; } if (*max < data[i]) { *max = data[i]; + *maxIndex = i; } - if (isNull(&lastVal, TSDB_DATA_TYPE_SMALLINT)) { - lastKey = primaryKey[i]; - lastVal = data[i]; - } else { - *wsum = lastVal * (primaryKey[i] - lastKey); - lastKey = primaryKey[i]; - lastVal = data[i]; - } + // if (isNull(&lastVal, TSDB_DATA_TYPE_SMALLINT)) { + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } else { + // *wsum = lastVal * (primaryKey[i] - lastKey); + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } } } static void getStatics_i32(int64_t *primaryKey, int32_t *data, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int64_t *wsum, int32_t *numOfNull) { + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { *min = INT64_MAX; *max = INT64_MIN; - *wsum = 0; + *minIndex = 0; + *maxIndex = 0; - int64_t lastKey = 0; - int32_t lastVal = TSDB_DATA_INT_NULL; + assert(numOfRow <= INT16_MAX); + + // int64_t lastKey = 0; + // int32_t lastVal = TSDB_DATA_INT_NULL; for (int32_t i = 0; i < numOfRow; ++i) { if (isNull(&data[i], TSDB_DATA_TYPE_INT)) { @@ -3728,28 +3724,33 @@ static void getStatics_i32(int64_t *primaryKey, int32_t *data, int32_t numOfRow, *sum += data[i]; if (*min > data[i]) { *min = data[i]; + *minIndex = i; } if (*max < data[i]) { *max = data[i]; + *maxIndex = i; } - if (isNull(&lastVal, TSDB_DATA_TYPE_INT)) { - lastKey = primaryKey[i]; - lastVal = data[i]; - } else { - *wsum = lastVal * (primaryKey[i] - lastKey); - lastKey = primaryKey[i]; - lastVal = data[i]; - } + // if (isNull(&lastVal, TSDB_DATA_TYPE_INT)) { + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } else { + // *wsum = lastVal * (primaryKey[i] - lastKey); + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } } } static void getStatics_i64(int64_t *primaryKey, int64_t *data, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int64_t *wsum, int32_t *numOfNull) { + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { *min = INT64_MAX; *max = INT64_MIN; - *wsum = 0; + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); int64_t lastKey = 0; int64_t lastVal = TSDB_DATA_BIGINT_NULL; @@ -3763,31 +3764,33 @@ static void getStatics_i64(int64_t *primaryKey, int64_t *data, int32_t numOfRow, *sum += data[i]; if (*min > data[i]) { *min = data[i]; + *minIndex = i; } if (*max < data[i]) { *max = data[i]; + *maxIndex = i; } - if (isNull(&lastVal, TSDB_DATA_TYPE_BIGINT)) { - lastKey = primaryKey[i]; - lastVal = data[i]; - } else { - *wsum = lastVal * (primaryKey[i] - lastKey); - lastKey = primaryKey[i]; - lastVal = data[i]; - } + // if (isNull(&lastVal, TSDB_DATA_TYPE_BIGINT)) { + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } else { + // *wsum = lastVal * (primaryKey[i] - lastKey); + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } } } static void getStatics_f(int64_t *primaryKey, float *data, int32_t numOfRow, double *min, double *max, double *sum, - double *wsum, int32_t *numOfNull) { + int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { *min = DBL_MAX; *max = -DBL_MAX; - *wsum = 0; + *minIndex = 0; + *maxIndex = 0; - int64_t lastKey = 0; - float lastVal = TSDB_DATA_FLOAT_NULL; + assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { if (isNull(&data[i], TSDB_DATA_TYPE_FLOAT)) { @@ -3798,28 +3801,33 @@ static void getStatics_f(int64_t *primaryKey, float *data, int32_t numOfRow, dou *sum += data[i]; if (*min > data[i]) { *min = data[i]; + *minIndex = i; } if (*max < data[i]) { *max = data[i]; + *maxIndex = i; } - if (isNull(&lastVal, TSDB_DATA_TYPE_FLOAT)) { - lastKey = primaryKey[i]; - lastVal = data[i]; - } else { - *wsum = lastVal * (primaryKey[i] - lastKey); - lastKey = primaryKey[i]; - lastVal = data[i]; - } + // if (isNull(&lastVal, TSDB_DATA_TYPE_FLOAT)) { + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } else { + // *wsum = lastVal * (primaryKey[i] - lastKey); + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } } } static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, double *min, double *max, double *sum, - double *wsum, int32_t *numOfNull) { + int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { *min = DBL_MAX; *max = -DBL_MAX; - *wsum = 0; + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); int64_t lastKey = 0; double lastVal = TSDB_DATA_DOUBLE_NULL; @@ -3833,25 +3841,27 @@ static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, do *sum += data[i]; if (*min > data[i]) { *min = data[i]; + *minIndex = i; } if (*max < data[i]) { *max = data[i]; + *maxIndex = i; } - if (isNull(&lastVal, TSDB_DATA_TYPE_DOUBLE)) { - lastKey = primaryKey[i]; - lastVal = data[i]; - } else { - *wsum = lastVal * (primaryKey[i] - lastKey); - lastKey = primaryKey[i]; - lastVal = data[i]; - } + // if (isNull(&lastVal, TSDB_DATA_TYPE_DOUBLE)) { + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } else { + // *wsum = lastVal * (primaryKey[i] - lastKey); + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } } } void getStatistics(char *priData, char *data, int32_t size, int32_t numOfRow, int32_t type, int64_t *min, int64_t *max, - int64_t *sum, int64_t *wsum, int32_t *numOfNull) { + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { int64_t *primaryKey = (int64_t *)priData; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { for (int32_t i = 0; i < numOfRow; ++i) { @@ -3862,403 +3872,186 @@ void getStatistics(char *priData, char *data, int32_t size, int32_t numOfRow, in } } else { if (type == TSDB_DATA_TYPE_TINYINT || type == TSDB_DATA_TYPE_BOOL) { - getStatics_i8(primaryKey, type, (int8_t *)data, numOfRow, min, max, sum, wsum, numOfNull); + getStatics_i8(primaryKey, type, (int8_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull); } else if (type == TSDB_DATA_TYPE_SMALLINT) { - getStatics_i16(primaryKey, (int16_t *)data, numOfRow, min, max, sum, wsum, numOfNull); + getStatics_i16(primaryKey, (int16_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull); } else if (type == TSDB_DATA_TYPE_INT) { - getStatics_i32(primaryKey, (int32_t *)data, numOfRow, min, max, sum, wsum, numOfNull); + getStatics_i32(primaryKey, (int32_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull); } else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) { - getStatics_i64(primaryKey, (int64_t *)data, numOfRow, min, max, sum, wsum, numOfNull); + getStatics_i64(primaryKey, (int64_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull); } else if (type == TSDB_DATA_TYPE_DOUBLE) { - getStatics_d(primaryKey, (double *)data, numOfRow, min, max, sum, wsum, numOfNull); + getStatics_d(primaryKey, (double *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull); } else if (type == TSDB_DATA_TYPE_FLOAT) { - getStatics_f(primaryKey, (float *)data, numOfRow, min, max, sum, wsum, numOfNull); + getStatics_f(primaryKey, (float *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull); } } } -void wavg_function_setup(SQLFunctionCtx *pCtx) { - memset(pCtx->aOutputBuf, 0, pCtx->outputBytes); - - pCtx->intermediateBuf[1].nType = TSDB_DATA_TYPE_TIMESTAMP; - pCtx->intermediateBuf[2].nType = -1; - - INIT_VAL(pCtx); -} - -bool wavg_function(SQLFunctionCtx *pCtx) { - void * pData = GET_INPUT_CHAR(pCtx); - int64_t *primaryKey = pCtx->ptsList; - assert(IS_DATA_BLOCK_LOADED(pCtx->blockStatus)); - /* assert(IS_INTER_BLOCK(pCtx->blockStatus)); */ - - int32_t notNullElems = 0; - - if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int64_t *retVal = pCtx->aOutputBuf; - int32_t *pDb = (int32_t *)pData; - int32_t i = 0; +/** + * param[1]: start time + * param[2]: end time + * @param pCtx + */ +static bool twa_function_setup(SQLFunctionCtx *pCtx) { + if (!function_setup(pCtx)) { + return false; + } - // Start diff in the block - for (; i < pCtx->size; ++i) { - assert(primaryKey[i] >= pCtx->intermediateBuf[1].i64Key); + SResultInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; + STwaInfo * pInfo = pResInfo->interResultBuf; - if (isNull(&pDb[i], TSDB_DATA_TYPE_INT)) continue; + pInfo->lastKey = INT64_MIN; + pInfo->type = pCtx->inputType; - if (pCtx->intermediateBuf[2].nType == -1) { - pCtx->intermediateBuf[2].i64Key = pDb[i]; - pCtx->intermediateBuf[2].nType = pCtx->inputType; - pCtx->intermediateBuf[1].i64Key = pCtx->nStartQueryTimestamp; - *retVal += pCtx->intermediateBuf[2].i64Key * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - } else { - *retVal += pCtx->intermediateBuf[2].i64Key * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - pCtx->intermediateBuf[2].i64Key = pDb[i]; - } + return true; +} +static FORCE_INLINE void setTWALastVal(SQLFunctionCtx *pCtx, const char *data, int32_t i, STwaInfo *pInfo) { + switch (pCtx->inputType) { + case TSDB_DATA_TYPE_INT: + pInfo->iLastValue = GET_INT32_VAL(data + pCtx->inputBytes * i); break; - } - - /* if (IS_INTER_BLOCK(pCtx->blockStatus)) { */ - /* *retVal += pCtx->preAggVals.wsum; */ - /* } */ - - for (++i; i < pCtx->size; i++) { - if (isNull(&pDb[i], TSDB_DATA_TYPE_INT)) continue; - - notNullElems++; - /* if (!IS_INTER_BLOCK(pCtx->blockStatus)) { */ - *retVal += pCtx->intermediateBuf[2].i64Key * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - /* } */ - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - pCtx->intermediateBuf[2].i64Key = pDb[i]; - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - int64_t *retVal = pCtx->aOutputBuf; - int64_t *pDb = (int64_t *)pData; - int32_t i = 0; - - // Start diff in the block - for (; i < pCtx->size; ++i) { - assert(primaryKey[i] >= pCtx->intermediateBuf[1].i64Key); - - if (isNull(&pDb[i], TSDB_DATA_TYPE_BIGINT)) continue; - - if (pCtx->intermediateBuf[2].nType == -1) { - pCtx->intermediateBuf[2].i64Key = pDb[i]; - pCtx->intermediateBuf[2].nType = pCtx->inputType; - pCtx->intermediateBuf[1].i64Key = pCtx->nStartQueryTimestamp; - *retVal += pCtx->intermediateBuf[2].i64Key * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - } else { - *retVal += pCtx->intermediateBuf[2].i64Key * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - pCtx->intermediateBuf[2].i64Key = pDb[i]; - } - + case TSDB_DATA_TYPE_TINYINT: + pInfo->iLastValue = GET_INT8_VAL(data + pCtx->inputBytes * i); break; - } - - /* if (IS_INTER_BLOCK(pCtx->blockStatus)) { */ - /* *retVal += pCtx->preAggVals.wsum; */ - /* } */ - - for (++i; i < pCtx->size; i++) { - if (isNull(&pDb[i], TSDB_DATA_TYPE_BIGINT)) continue; - - notNullElems++; - /* if (!IS_INTER_BLOCK(pCtx->blockStatus)) { */ - *retVal += pCtx->intermediateBuf[2].i64Key * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - /* } */ - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - pCtx->intermediateBuf[2].i64Key = pDb[i]; - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *retVal = pCtx->aOutputBuf; - double *pDb = (double *)pData; - int32_t i = 0; - - // Start diff in the block - for (; i < pCtx->size; ++i) { - assert(primaryKey[i] >= pCtx->intermediateBuf[1].i64Key); - - if (isNull(&pDb[i], TSDB_DATA_TYPE_DOUBLE)) continue; - - if (pCtx->intermediateBuf[2].nType == -1) { - pCtx->intermediateBuf[2].dKey = pDb[i]; - pCtx->intermediateBuf[2].nType = pCtx->inputType; - pCtx->intermediateBuf[1].i64Key = pCtx->nStartQueryTimestamp; - *retVal += pCtx->intermediateBuf[2].dKey * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - } else { - *retVal += pCtx->intermediateBuf[2].dKey * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - pCtx->intermediateBuf[2].dKey = pDb[i]; - } - + case TSDB_DATA_TYPE_SMALLINT: + pInfo->iLastValue = GET_INT16_VAL(data + pCtx->inputBytes * i); break; - } - - /* if (IS_INTER_BLOCK(pCtx->blockStatus)) { */ - /* *retVal += *(double *)(&(pCtx->preAggVals.wsum)); */ - /* } */ - - for (++i; i < pCtx->size; i++) { - if (isNull(&pDb[i], TSDB_DATA_TYPE_DOUBLE)) continue; - - notNullElems++; - *retVal += pCtx->intermediateBuf[2].dKey * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - pCtx->intermediateBuf[2].dKey = pDb[i]; - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = pCtx->aOutputBuf; - float * pDb = (float *)pData; - int32_t i = 0; - - // Start diff in the block - for (; i < pCtx->size; ++i) { - assert(primaryKey[i] >= pCtx->intermediateBuf[1].i64Key); - - if (isNull(&pDb[i], TSDB_DATA_TYPE_FLOAT)) continue; - - if (pCtx->intermediateBuf[2].nType == -1) { - pCtx->intermediateBuf[2].dKey = pDb[i]; - pCtx->intermediateBuf[2].nType = pCtx->inputType; - pCtx->intermediateBuf[1].i64Key = pCtx->nStartQueryTimestamp; - *retVal += pCtx->intermediateBuf[2].dKey * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - } else { - *retVal += pCtx->intermediateBuf[2].dKey * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - pCtx->intermediateBuf[2].dKey = pDb[i]; - } - + case TSDB_DATA_TYPE_BIGINT: + pInfo->iLastValue = GET_INT64_VAL(data + pCtx->inputBytes * i); break; - } - - /* if (IS_INTER_BLOCK(pCtx->blockStatus)) { */ - /* *retVal += *(double *)(&(pCtx->preAggVals.wsum)); */ - /* } */ - - for (++i; i < pCtx->size; i++) { - if (isNull(&pDb[i], TSDB_DATA_TYPE_FLOAT)) continue; - - notNullElems++; - *retVal += pCtx->intermediateBuf[2].dKey * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - pCtx->intermediateBuf[2].dKey = pDb[i]; - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - int64_t *retVal = pCtx->aOutputBuf; - int16_t *pDb = (int16_t *)pData; - int32_t i = 0; - - // Start diff in the block - for (; i < pCtx->size; ++i) { - assert(primaryKey[i] >= pCtx->intermediateBuf[1].i64Key); - - if (isNull(&pDb[i], TSDB_DATA_TYPE_SMALLINT)) continue; - - if (pCtx->intermediateBuf[2].nType == -1) { - pCtx->intermediateBuf[2].i64Key = pDb[i]; - pCtx->intermediateBuf[2].nType = pCtx->inputType; - pCtx->intermediateBuf[1].i64Key = pCtx->nStartQueryTimestamp; - *retVal += pCtx->intermediateBuf[2].i64Key * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - } else { - *retVal += pCtx->intermediateBuf[2].i64Key * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - pCtx->intermediateBuf[2].i64Key = pDb[i]; - } - + case TSDB_DATA_TYPE_FLOAT: + pInfo->dLastValue = GET_FLOAT_VAL(data + pCtx->inputBytes * i); break; - } - - for (++i; i < pCtx->size; i++) { - if (isNull(&pDb[i], TSDB_DATA_TYPE_SMALLINT)) continue; - - notNullElems++; - *retVal += pCtx->intermediateBuf[2].i64Key * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - pCtx->intermediateBuf[2].i64Key = pDb[i]; - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - int64_t *retVal = pCtx->aOutputBuf; - int8_t * pDb = (int8_t *)pData; - int32_t i = 0; - - // Start diff in the block - for (; i < pCtx->size; ++i) { - assert(primaryKey[i] >= pCtx->intermediateBuf[1].i64Key); - - if (isNull((char *)&pDb[i], TSDB_DATA_TYPE_TINYINT)) continue; - - if (pCtx->intermediateBuf[2].nType == -1) { - pCtx->intermediateBuf[2].i64Key = pDb[i]; - pCtx->intermediateBuf[2].nType = pCtx->inputType; - pCtx->intermediateBuf[1].i64Key = pCtx->nStartQueryTimestamp; - *retVal += pCtx->intermediateBuf[2].i64Key * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - } else { - *retVal += pCtx->intermediateBuf[2].i64Key * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - pCtx->intermediateBuf[2].i64Key = pDb[i]; - } - + case TSDB_DATA_TYPE_DOUBLE: + pInfo->dLastValue = GET_DOUBLE_VAL(data + pCtx->inputBytes * i); break; - } + default: + assert(0); + } +} - for (++i; i < pCtx->size; i++) { - if (isNull((char *)&pDb[i], TSDB_DATA_TYPE_TINYINT)) continue; +static void twa_function(SQLFunctionCtx *pCtx) { + void * data = GET_INPUT_CHAR(pCtx); + TSKEY *primaryKey = pCtx->ptsList; - notNullElems++; - *retVal += pCtx->intermediateBuf[2].i64Key * (primaryKey[i] - pCtx->intermediateBuf[1].i64Key); - pCtx->intermediateBuf[1].i64Key = primaryKey[i]; - pCtx->intermediateBuf[2].i64Key = pDb[i]; - } - } + assert(IS_DATA_BLOCK_LOADED(pCtx->blockStatus)); - pCtx->numOfIteratedElems += notNullElems; + int32_t notNullElems = 0; - return true; -} + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + STwaInfo * pInfo = pResInfo->interResultBuf; -bool wavg_function_f(SQLFunctionCtx *pCtx, int32_t index) { - // TODO : - return false; -} -void wavg_function_finalize(SQLFunctionCtx *pCtx) { - if (pCtx->intermediateBuf[2].nType == -1) { - *((double *)(pCtx->aOutputBuf)) = TSDB_DATA_DOUBLE_NULL; - SET_VAL(pCtx, 0, 0); - return; - } + int32_t i = 0; - assert(pCtx->intermediateBuf[3].i64Key >= pCtx->intermediateBuf[1].i64Key); - if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - int64_t *retVal = pCtx->aOutputBuf; - *retVal += pCtx->intermediateBuf[2].i64Key * (pCtx->intermediateBuf[3].i64Key - pCtx->intermediateBuf[1].i64Key); - *(double *)pCtx->aOutputBuf = (*retVal) / (double)(pCtx->intermediateBuf[3].i64Key - pCtx->nStartQueryTimestamp); - } else { - double *retVal = pCtx->aOutputBuf; - *retVal += pCtx->intermediateBuf[2].dKey * (pCtx->intermediateBuf[3].i64Key - pCtx->intermediateBuf[1].i64Key); - *retVal = *retVal / (pCtx->intermediateBuf[3].i64Key - pCtx->nStartQueryTimestamp); + // skip null value + while (pCtx->hasNull && i < pCtx->size && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { + i++; } - SET_VAL(pCtx, 1, 1); -} -static bool wavg_dist_function(SQLFunctionCtx *pCtx) { - void * pData = GET_INPUT_CHAR(pCtx); - int64_t *primaryKey = pCtx->ptsList; - assert(IS_DATA_BLOCK_LOADED(pCtx->blockStatus)); + if (i >= pCtx->size) { + return; + } - SWavgRuntime *output = pCtx->aOutputBuf; + if (pInfo->lastKey == INT64_MIN) { + pInfo->lastKey = pCtx->nStartQueryTimestamp; + setTWALastVal(pCtx, data, i, pInfo); - output->type = pCtx->inputType; - int32_t notNullElems = 0; + pInfo->hasResult = DATA_SET_FLAG; + } - if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *pDb = (int32_t *)pData; + notNullElems++; - for (int32_t i = 0; i < pCtx->size; i++) { - if (isNull(&pDb[i], TSDB_DATA_TYPE_INT)) continue; + if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { + pInfo->dOutput += pInfo->dLastValue * (primaryKey[i] - pInfo->lastKey); + } else { + pInfo->iOutput += pInfo->iLastValue * (primaryKey[i] - pInfo->lastKey); + } - notNullElems++; - SET_HAS_DATA_FLAG(output->valFlag); + pInfo->lastKey = primaryKey[i]; + setTWALastVal(pCtx, data, i, pInfo); - output->iOutput += output->iLastValue * (primaryKey[i] - output->lastKey); - output->lastKey = primaryKey[i]; - output->iLastValue = pDb[i]; + for (++i; i < pCtx->size; i++) { + if (pCtx->hasNull && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { + continue; } - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - int64_t *pDb = (int64_t *)pData; - - for (int32_t i = 0; i < pCtx->size; i++) { - if (isNull(&pDb[i], TSDB_DATA_TYPE_BIGINT)) continue; - - notNullElems++; - SET_HAS_DATA_FLAG(output->valFlag); - - output->iOutput += output->iLastValue * (primaryKey[i] - output->lastKey); - output->lastKey = primaryKey[i]; - output->iLastValue = pDb[i]; + notNullElems++; + if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { + pInfo->dOutput += pInfo->dLastValue * (primaryKey[i] - pInfo->lastKey); + } else { + pInfo->iOutput += pInfo->iLastValue * (primaryKey[i] - pInfo->lastKey); } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *pDb = (double *)pData; + pInfo->lastKey = primaryKey[i]; + setTWALastVal(pCtx, data, i, pInfo); + } - for (int32_t i = 0; i < pCtx->size; i++) { - if (isNull(&pDb[i], TSDB_DATA_TYPE_BIGINT)) continue; + SET_VAL(pCtx, notNullElems, 1); - notNullElems++; - SET_HAS_DATA_FLAG(output->valFlag); + if (notNullElems > 0) { + pResInfo->hasResult = DATA_SET_FLAG; + } - output->dOutput += output->dLastValue * (primaryKey[i] - output->lastKey); - output->lastKey = primaryKey[i]; - output->dLastValue = pDb[i]; - } + if (pResInfo->superTableQ) { + memcpy(pCtx->aOutputBuf, pInfo, sizeof(STwaInfo)); + } - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - float *pDb = (float *)pData; + // pCtx->numOfIteratedElems += notNullElems; +} - for (int32_t i = 0; i < pCtx->size; i++) { - if (isNull(&pDb[i], TSDB_DATA_TYPE_FLOAT)) continue; +static bool twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { + void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return true; + } - notNullElems++; - SET_HAS_DATA_FLAG(output->valFlag); + SET_VAL(pCtx, 1, 1); - output->dOutput += output->dLastValue * (primaryKey[i] - output->lastKey); - output->lastKey = primaryKey[i]; - output->dLastValue = pDb[i]; - } + TSKEY *primaryKey = pCtx->ptsList; - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - int16_t *pDb = (int16_t *)pData; + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + STwaInfo * pInfo = pResInfo->interResultBuf; - for (int32_t i = 0; i < pCtx->size; i++) { - if (isNull(&pDb[i], TSDB_DATA_TYPE_SMALLINT)) continue; + if (pInfo->lastKey == INT64_MIN) { + pInfo->lastKey = pCtx->nStartQueryTimestamp; + setTWALastVal(pCtx, pData, 0, pInfo); - notNullElems++; - SET_HAS_DATA_FLAG(output->valFlag); + pInfo->hasResult = DATA_SET_FLAG; + } - output->iOutput += output->iLastValue * (primaryKey[i] - output->lastKey); - output->lastKey = primaryKey[i]; - output->iLastValue = pDb[i]; - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - int8_t *pDb = (int8_t *)pData; + if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { + pInfo->dOutput += pInfo->dLastValue * (primaryKey[index] - pInfo->lastKey); + } else { + pInfo->iOutput += pInfo->iLastValue * (primaryKey[index] - pInfo->lastKey); + } - for (int32_t i = 0; i < pCtx->size; i++) { - if (isNull((char *)&pDb[i], TSDB_DATA_TYPE_TINYINT)) continue; + // record the last key/value + pInfo->lastKey = primaryKey[index]; + setTWALastVal(pCtx, pData, 0, pInfo); - notNullElems++; - SET_HAS_DATA_FLAG(output->valFlag); + // pCtx->numOfIteratedElems += 1; + pResInfo->hasResult = DATA_SET_FLAG; - output->iOutput += output->iLastValue * (primaryKey[i] - output->lastKey); - output->lastKey = primaryKey[i]; - output->iLastValue = pDb[i]; - } + if (pResInfo->superTableQ) { + memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(STwaInfo)); } - SET_VAL(pCtx, notNullElems, 1); return true; } -static bool wavg_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { return false; } +static void twa_func_merge(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pResInfo->superTableQ); -static void wavg_dist_merge(SQLFunctionCtx *pCtx) { - SWavgRuntime *pBuf = (SWavgRuntime *)pCtx->aOutputBuf; - char * indicator = pCtx->aInputElemBuf; + STwaInfo *pBuf = (STwaInfo *)pCtx->aOutputBuf; + char * indicator = pCtx->aInputElemBuf; int32_t numOfNotNull = 0; - for (int32_t i = 0; i < pCtx->size; ++i) { - SWavgRuntime *pInput = indicator; + for (int32_t i = 0; i < pCtx->size; ++i, indicator += sizeof(STwaInfo)) { + STwaInfo *pInput = indicator; - if (!HAS_DATA_FLAG(pInput->valFlag)) { - indicator += sizeof(SWavgRuntime); + if (pInput->hasResult != DATA_SET_FLAG) { continue; } @@ -4269,32 +4062,55 @@ static void wavg_dist_merge(SQLFunctionCtx *pCtx) { pBuf->dOutput += pInput->dOutput; } - pBuf->sKey = pInput->sKey; - pBuf->eKey = pInput->eKey; + pBuf->SKey = pInput->SKey; + pBuf->EKey = pInput->EKey; pBuf->lastKey = pInput->lastKey; pBuf->iLastValue = pInput->iLastValue; } SET_VAL(pCtx, numOfNotNull, 1); + + if (numOfNotNull > 0) { + pBuf->hasResult = DATA_SET_FLAG; + } +} + +/* + * To copy the input to interResBuf to avoid the input buffer space be over writen + * by next input data. The TWA function only applies to each table, so no merge procedure + * is required, we simply copy to the resut ot interResBuffer. + */ +void twa_function_copy(SQLFunctionCtx *pCtx) { + assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + + memcpy(pResInfo->interResultBuf, pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); + pResInfo->hasResult = ((STwaInfo *)pCtx->aInputElemBuf)->hasResult; } -static void wavg_dist_second_merge(SQLFunctionCtx *pCtx) { - SWavgRuntime *pWavg = (SWavgRuntime *)pCtx->aInputElemBuf; +void twa_function_finalizer(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + + STwaInfo *pInfo = (STwaInfo *)pResInfo->interResultBuf; + assert(pInfo->EKey >= pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult); - if (!HAS_DATA_FLAG(pWavg->valFlag)) { - *((int64_t *)(pCtx->aOutputBuf)) = TSDB_DATA_DOUBLE_NULL; - SET_VAL(pCtx, 0, 0); + if (pInfo->hasResult != DATA_SET_FLAG) { + setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); return; } - if (pWavg->type >= TSDB_DATA_TYPE_TINYINT && pWavg->type <= TSDB_DATA_TYPE_BIGINT) { - *(double *)pCtx->aOutputBuf = - (pWavg->iOutput + pWavg->iLastValue * (pWavg->eKey - pWavg->lastKey)) / (double)(pWavg->eKey - pWavg->sKey); + if (pInfo->SKey == pInfo->EKey) { + *(double *)pCtx->aOutputBuf = 0; + } else if (pInfo->type >= TSDB_DATA_TYPE_TINYINT && pInfo->type <= TSDB_DATA_TYPE_BIGINT) { + pInfo->iOutput += pInfo->iLastValue * (pInfo->EKey - pInfo->lastKey); + *(double *)pCtx->aOutputBuf = pInfo->iOutput / (double)(pInfo->EKey - pInfo->SKey); } else { - *(double *)pCtx->aOutputBuf = - (pWavg->dOutput + pWavg->dLastValue * (pWavg->eKey - pWavg->lastKey)) / (pWavg->eKey - pWavg->sKey); + pInfo->dOutput += pInfo->dLastValue * (pInfo->EKey - pInfo->lastKey); + *(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->EKey - pInfo->SKey); } - SET_VAL(pCtx, 1, 1); + + GET_RES_INFO(pCtx)->numOfRes = 1; + resetResultInfo(GET_RES_INFO(pCtx)); } /** @@ -4302,15 +4118,10 @@ static void wavg_dist_second_merge(SQLFunctionCtx *pCtx) { * param[2]: next value of specified timestamp * param[3]: denotes if the result is a precious result or interpolation results * - * intermediate[0]: interpolation type - * intermediate[1]: precious specified timestamp, the pCtx->startTimetamp is changed during query to satisfy the query procedure - * intermediate[2]: flag that denotes if it is a primary timestamp column or not - * intermediate[3]: tags. reserved for tags, the function is available for stable query, so the intermediate[3] must be reserved. - * * @param pCtx */ -static bool interp_function(SQLFunctionCtx *pCtx) { - /* at this point, the value is existed, return directly */ +static void interp_function(SQLFunctionCtx *pCtx) { + // at this point, the value is existed, return directly if (pCtx->param[3].i64Key == 1) { char *pData = GET_INPUT_CHAR(pCtx); assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); @@ -4320,126 +4131,77 @@ static bool interp_function(SQLFunctionCtx *pCtx) { * Note: the result of primary timestamp column uses the timestamp specified by user in the query sql */ assert(pCtx->param[3].i64Key == 2); - int32_t interpoType = pCtx->intermediateBuf[0].i64Key; - if (interpoType == TSDB_INTERPO_NONE) { - /* set no output result */ + SInterpInfo interpInfo = *(SInterpInfo *)pCtx->aOutputBuf; + SInterpInfoDetail *pInfoDetail = interpInfo.pInterpDetail; + + /* set no output result */ + if (pInfoDetail->type == TSDB_INTERPO_NONE) { pCtx->param[3].i64Key = 0; - } else if (pCtx->intermediateBuf[2].i64Key == 1) { - *(TSKEY *)pCtx->aOutputBuf = pCtx->intermediateBuf[1].i64Key; + } else if (pInfoDetail->primaryCol == 1) { + *(TSKEY *)pCtx->aOutputBuf = pInfoDetail->ts; } else { - if (interpoType == TSDB_INTERPO_NULL) { + if (pInfoDetail->type == TSDB_INTERPO_NULL) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); - } else if (interpoType == TSDB_INTERPO_SET_VALUE) { + } else if (pInfoDetail->type == TSDB_INTERPO_SET_VALUE) { tVariantDump(&pCtx->param[1], pCtx->aOutputBuf, pCtx->inputType); - } else if (interpoType == TSDB_INTERPO_PREV) { - if (strcmp(pCtx->param[1].pz, TSDB_DATA_NULL_STR_L) == 0) { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); - goto _end; - } - + } else if (pInfoDetail->type == TSDB_INTERPO_PREV) { char *data = pCtx->param[1].pz; - char *pVal = NULL; - if (pCtx->param[1].nType == TSDB_DATA_TYPE_BINARY) { - pVal = strsep(&data, ","); - pVal = strsep(&data, ","); - } else { - wchar_t *token = NULL; - pVal = wcstok(data, L",", &token); - pVal = wcstok(NULL, L",", &token); - } + char *pVal = data + TSDB_KEYSIZE; - if ((pCtx->outputType >= TSDB_DATA_TYPE_BOOL && pCtx->outputType <= TSDB_DATA_TYPE_BIGINT) || - pCtx->outputType == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t v = strtoll(pVal, NULL, 10); + if (pCtx->outputType == TSDB_DATA_TYPE_FLOAT) { + float v = GET_DOUBLE_VAL(pVal); assignVal(pCtx->aOutputBuf, &v, pCtx->outputBytes, pCtx->outputType); - } else if (pCtx->outputType == TSDB_DATA_TYPE_FLOAT) { - float v = (float)strtod(pVal, NULL); - if (isNull(&v, pCtx->outputType)) { - setNull(pCtx->aOutputBuf, pCtx->inputType, pCtx->inputBytes); - } else { - assignVal(pCtx->aOutputBuf, &v, pCtx->outputBytes, pCtx->outputType); - } - } else if (pCtx->outputType == TSDB_DATA_TYPE_DOUBLE) { - double v = strtod(pVal, NULL); - if (isNull(&v, pCtx->outputType)) { - setNull(pCtx->aOutputBuf, pCtx->inputType, pCtx->inputBytes); - } else { - assignVal(pCtx->aOutputBuf, &v, pCtx->outputBytes, pCtx->outputType); - } - } else if (pCtx->outputType == TSDB_DATA_TYPE_BINARY) { - assignVal(pCtx->aOutputBuf, pVal, pCtx->outputBytes, pCtx->outputType); - } else if (pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { + } else { assignVal(pCtx->aOutputBuf, pVal, pCtx->outputBytes, pCtx->outputType); } - } else if (interpoType == TSDB_INTERPO_LINEAR) { - if (strcmp(pCtx->param[1].pz, TSDB_DATA_NULL_STR_L) == 0) { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); - goto _end; - } - + } else if (pInfoDetail->type == TSDB_INTERPO_LINEAR) { char *data1 = pCtx->param[1].pz; char *data2 = pCtx->param[2].pz; - char *pTimestamp1 = strsep(&data1, ","); - char *pTimestamp2 = strsep(&data2, ","); + char *pVal1 = data1 + TSDB_KEYSIZE; + char *pVal2 = data2 + TSDB_KEYSIZE; - char *pVal1 = strsep(&data1, ","); - char *pVal2 = strsep(&data2, ","); + SPoint point1 = {.key = *(TSKEY *)data1, .val = &pCtx->param[1].i64Key}; + SPoint point2 = {.key = *(TSKEY *)data2, .val = &pCtx->param[2].i64Key}; - SPoint point1 = {.key = strtol(pTimestamp1, NULL, 10), .val = &pCtx->param[1].i64Key}; - SPoint point2 = {.key = strtol(pTimestamp2, NULL, 10), .val = &pCtx->param[2].i64Key}; - - SPoint point = {.key = pCtx->intermediateBuf[1].i64Key, .val = pCtx->aOutputBuf}; + SPoint point = {.key = pInfoDetail->ts, .val = pCtx->aOutputBuf}; int32_t srcType = pCtx->inputType; if ((srcType >= TSDB_DATA_TYPE_TINYINT && srcType <= TSDB_DATA_TYPE_BIGINT) || - srcType == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t v1 = strtol(pVal1, NULL, 10); - point1.val = &v1; + srcType == TSDB_DATA_TYPE_TIMESTAMP || srcType == TSDB_DATA_TYPE_DOUBLE) { + point1.val = pVal1; - int64_t v2 = strtol(pVal2, NULL, 10); - point2.val = &v2; + point2.val = pVal2; - if (isNull(&v1, srcType) || isNull(&v2, srcType)) { + if (isNull(pVal1, srcType) || isNull(pVal2, srcType)) { setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); } else { taosDoLinearInterpolation(pCtx->outputType, &point1, &point2, &point); } } else if (srcType == TSDB_DATA_TYPE_FLOAT) { - float v1 = strtod(pVal1, NULL); - point1.val = &v1; - - float v2 = strtod(pVal2, NULL); - point2.val = &v2; + float v1 = GET_DOUBLE_VAL(pVal1); + float v2 = GET_DOUBLE_VAL(pVal2); - if (isNull(&v1, srcType) || isNull(&v2, srcType)) { - setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); - } else { - taosDoLinearInterpolation(pCtx->outputType, &point1, &point2, &point); - } - } else if (srcType == TSDB_DATA_TYPE_DOUBLE) { - double v1 = strtod(pVal1, NULL); point1.val = &v1; - - double v2 = strtod(pVal2, NULL); point2.val = &v2; - if (isNull(&v1, srcType) || isNull(&v2, srcType)) { + if (isNull(pVal1, srcType) || isNull(pVal2, srcType)) { setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); } else { taosDoLinearInterpolation(pCtx->outputType, &point1, &point2, &point); } - } else if (srcType == TSDB_DATA_TYPE_BOOL || srcType == TSDB_DATA_TYPE_BINARY || - srcType == TSDB_DATA_TYPE_NCHAR) { + + } else { setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); } } } + + free(interpInfo.pInterpDetail); } -_end: pCtx->size = pCtx->param[3].i64Key; tVariantDestroy(&pCtx->param[1]); @@ -4447,57 +4209,126 @@ _end: // data in the check operation are all null, not output SET_VAL(pCtx, pCtx->size, 1); - return false; +} + +static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) { + if (!function_setup(pCtx)) { + return false; // not initialized since it has been initialized + } + + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + STSCompInfo *pInfo = pResInfo->interResultBuf; + + pInfo->pTSBuf = tsBufCreate(false); + pInfo->pTSBuf->tsOrder = pCtx->order; + return true; +} + +static void ts_comp_function(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + STSBuf * pTSbuf = ((STSCompInfo *)(pResInfo->interResultBuf))->pTSBuf; + + const char *input = GET_INPUT_CHAR(pCtx); + + // primary ts must be existed, so no need to check its existance + if (pCtx->order == TSQL_SO_ASC) { + tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, input, pCtx->size * TSDB_KEYSIZE); + } else { + for (int32_t i = pCtx->size - 1; i >= 0; --i) { + char *d = GET_INPUT_CHAR_INDEX(pCtx, i); + tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, d, TSDB_KEYSIZE); + } + } + + SET_VAL(pCtx, pCtx->size, 1); + + pResInfo->hasResult = DATA_SET_FLAG; +} + +static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { + void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; + } + + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + STSCompInfo *pInfo = pResInfo->interResultBuf; + + STSBuf *pTSbuf = pInfo->pTSBuf; + + tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, pData, TSDB_KEYSIZE); + SET_VAL(pCtx, pCtx->size, 1); + + pResInfo->hasResult = DATA_SET_FLAG; +} + +static void ts_comp_finalize(SQLFunctionCtx *pCtx) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + + STSCompInfo *pInfo = pResInfo->interResultBuf; + STSBuf * pTSbuf = pInfo->pTSBuf; + + tsBufFlush(pTSbuf); + strcpy(pCtx->aOutputBuf, pTSbuf->path); + + tsBufDestory(pTSbuf); + resetResultInfo(GET_RES_INFO(pCtx)); } /* - * function with the same value is compatible in selection clause - * Note: tag function, ts function is not need to check the compatible with other functions + * function compatible list. + * tag and ts are not involved in the compatibility check + * + * 1. functions that are not simultaneously present with any other functions. e.g., + * diff/ts_z/top/bottom + * 2. functions that are only allowed to be present only with same functions. e.g., last_row, interp + * 3. functions that are allowed to be present with other functions. + * e.g., count/sum/avg/min/max/stddev/percentile/apercentile/first/last... * - * top/bottom is the last one */ -int32_t funcCompatList[36] = { - /* count, sum, avg, min, max, stddev, percentile, apercentile, first, - last, last_row, leastsqr, */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 7, 1, - - /* top, bottom, spread, wavg, ts, ts_dummy, tag, colprj, tagprj, - arithmetic, diff, */ - 2, 5, 1, 1, 1, 1, 1, 3, 3, 3, 4, +int32_t funcCompatDefList[28] = { + /* + * count, sum, avg, min, max, stddev, percentile, apercentile, first, last + */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - /*sum_d, avg_d, min_d, max_d, first_d, last_d, last_row_d, spread_dst, - wavg_dst, top_dst, bottom_dst, */ - 1, 1, 1, 1, 1, 1, 7, 1, 1, 2, 5, + /* + * last_row, top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_z, tag + */ + 4, -1, -1, 1, 1, 1, 1, 1, 1, -1, 1, - /*apercentile_dst, interp*/ - 1, 6, -}; + /* + * colprj, tagprj, arithmetic, diff, first_dist, last_dist, interp + */ + 1, 1, 1, -1, 1, 1, 5}; -SQLAggFuncElem aAggs[36] = { +SQLAggFuncElem aAggs[28] = { { - // 0 + // 0, count function does not invoke the finalize function "count", TSDB_FUNC_COUNT, TSDB_FUNC_COUNT, TSDB_BASE_FUNC_SO, function_setup, count_function, count_function_f, - no_next_step, noop, count_dist_merge, count_dist_merge, count_load_data_info, + no_next_step, noop, count_func_merge, count_func_merge, count_load_data_info, }, { // 1 - "sum", TSDB_FUNC_SUM, TSDB_FUNC_SUM_DST, TSDB_BASE_FUNC_SO, function_setup, sum_function, sum_function_f, - no_next_step, function_finalize, noop, noop, precal_req_load_info, + "sum", TSDB_FUNC_SUM, TSDB_FUNC_SUM, TSDB_BASE_FUNC_SO, function_setup, sum_function, sum_function_f, + no_next_step, function_finalizer, sum_func_merge, sum_func_second_merge, precal_req_load_info, }, { // 2 - "avg", TSDB_FUNC_AVG, TSDB_FUNC_AVG_DST, TSDB_BASE_FUNC_SO, function_setup, sum_function, sum_function_f, - no_next_step, avg_finalizer, noop, noop, precal_req_load_info, + "avg", TSDB_FUNC_AVG, TSDB_FUNC_AVG, TSDB_BASE_FUNC_SO, function_setup, avg_function, avg_function_f, + no_next_step, avg_finalizer, avg_func_merge, avg_func_second_merge, precal_req_load_info, }, { // 3 - "min", TSDB_FUNC_MIN, TSDB_FUNC_MIN_DST, TSDB_BASE_FUNC_SO, min_function_setup, min_function, min_function_f, - no_next_step, function_finalize, noop, noop, precal_req_load_info, + "min", TSDB_FUNC_MIN, TSDB_FUNC_MIN, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, min_func_setup, + min_function, min_function_f, no_next_step, function_finalizer, min_func_merge, min_func_second_merge, + precal_req_load_info, }, { // 4 - "max", TSDB_FUNC_MAX, TSDB_FUNC_MAX_DST, TSDB_BASE_FUNC_SO, max_function_setup, max_function, max_function_f, - no_next_step, function_finalize, noop, noop, precal_req_load_info, + "max", TSDB_FUNC_MAX, TSDB_FUNC_MAX, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, max_func_setup, + max_function, max_function_f, no_next_step, function_finalizer, max_func_merge, max_func_second_merge, + precal_req_load_info, }, { // 5 @@ -4513,54 +4344,61 @@ SQLAggFuncElem aAggs[36] = { }, { // 7 - "apercentile", TSDB_FUNC_APERCT, TSDB_FUNC_APERCT_DST, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, function_setup, apercentile_function, - apercentile_function_f, no_next_step, apercentile_finalizer, noop, noop, data_req_load_info, + "apercentile", TSDB_FUNC_APERCT, TSDB_FUNC_APERCT, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC, + apercentile_function_setup, apercentile_function, apercentile_function_f, no_next_step, apercentile_finalizer, + apercentile_func_merge, apercentile_func_second_merge, data_req_load_info, }, { // 8 - "first", TSDB_FUNC_FIRST, TSDB_FUNC_FIRST_DST, TSDB_BASE_FUNC_SO, function_setup, first_function, - first_function_f, no_next_step, function_finalize, noop, noop, first_data_req_info, + "first", TSDB_FUNC_FIRST, TSDB_FUNC_FIRST_DST, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, function_setup, + first_function, first_function_f, no_next_step, function_finalizer, noop, noop, first_data_req_info, }, { // 9 - "last", TSDB_FUNC_LAST, TSDB_FUNC_LAST_DST, TSDB_BASE_FUNC_SO, function_setup, last_function, last_function_f, - no_next_step, function_finalize, noop, noop, last_data_req_info, + "last", TSDB_FUNC_LAST, TSDB_FUNC_LAST_DST, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, function_setup, + last_function, last_function_f, no_next_step, function_finalizer, noop, noop, last_data_req_info, }, { // 10 - "last_row", TSDB_FUNC_LAST_ROW, TSDB_FUNC_LAST_ROW_DST, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC, function_setup, interp_function, noop, - no_next_step, noop, noop, copy_function, no_data_info, + "last_row", TSDB_FUNC_LAST_ROW, TSDB_FUNC_LAST_ROW, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS | + TSDB_FUNCSTATE_SELECTIVITY, + first_last_function_setup, last_row_function, noop, no_next_step, last_row_finalizer, noop, + last_dist_func_second_merge, data_req_load_info, }, { // 11 - "leastsquares", TSDB_FUNC_LEASTSQR, TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, leastsquares_function_setup, - leastsquares_function, leastsquares_function_f, no_next_step, leastsquare_finalizer, noop, noop, - data_req_load_info, + "top", TSDB_FUNC_TOP, TSDB_FUNC_TOP, TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF | + TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, + top_bottom_function_setup, top_function, top_function_f, no_next_step, top_bottom_func_finalizer, + top_func_merge, top_func_second_merge, data_req_load_info, }, { // 12 - "top", TSDB_FUNC_TOP, TSDB_FUNC_TOP_DST, TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS, - top_bottom_function_setup, top_function, top_function_f, no_next_step, top_bottom_function_finalizer, noop, - noop, data_req_load_info, + "bottom", TSDB_FUNC_BOTTOM, TSDB_FUNC_BOTTOM, TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF | + TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, + top_bottom_function_setup, bottom_function, bottom_function_f, no_next_step, top_bottom_func_finalizer, + bottom_func_merge, bottom_func_second_merge, data_req_load_info, }, { // 13 - "bottom", TSDB_FUNC_BOTTOM, TSDB_FUNC_BOTTOM_DST, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS, top_bottom_function_setup, bottom_function, - bottom_function_f, no_next_step, top_bottom_function_finalizer, noop, noop, data_req_load_info, + "spread", TSDB_FUNC_SPREAD, TSDB_FUNC_SPREAD, TSDB_BASE_FUNC_SO, spread_function_setup, spread_function, + spread_function_f, no_next_step, spread_function_finalizer, spread_func_merge, spread_func_sec_merge, + count_load_data_info, }, { // 14 - "spread", TSDB_FUNC_SPREAD, TSDB_FUNC_SPREAD_DST, TSDB_BASE_FUNC_SO, spread_function_setup, spread_function, - spread_function_f, no_next_step, spread_function_finalize, noop, noop, count_load_data_info, + "twa", TSDB_FUNC_TWA, TSDB_FUNC_TWA, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, twa_function_setup, + twa_function, twa_function_f, no_next_step, twa_function_finalizer, twa_func_merge, twa_function_copy, + data_req_load_info, }, { // 15 - "wavg", TSDB_FUNC_WAVG, TSDB_FUNC_WAVG_DST, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, wavg_function_setup, - wavg_function, wavg_function_f, no_next_step, wavg_function_finalize, noop, noop, data_req_load_info, + "leastsquares", TSDB_FUNC_LEASTSQR, TSDB_FUNC_INVALID_ID, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, leastsquares_function_setup, + leastsquares_function, leastsquares_function_f, no_next_step, leastsquares_finalizer, noop, noop, + data_req_load_info, }, { // 16 @@ -4571,119 +4409,66 @@ SQLAggFuncElem aAggs[36] = { { // 17 "ts", TSDB_FUNC_TS_DUMMY, TSDB_FUNC_TS_DUMMY, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, function_setup, noop, - noop, no_next_step, noop, copy_function, copy_function, no_data_info, + noop, no_next_step, noop, copy_function, copy_function, data_req_load_info, }, { // 18 - "tag", TSDB_FUNC_TAG, TSDB_FUNC_TAG, TSDB_BASE_FUNC_SO, function_setup, tag_function, tag_function_f, + "tag", TSDB_FUNC_TAG_DUMMY, TSDB_FUNC_TAG_DUMMY, TSDB_BASE_FUNC_SO, function_setup, tag_function, noop, no_next_step, noop, copy_function, copy_function, no_data_info, }, - // column project sql function { // 19 - "colprj", TSDB_FUNC_PRJ, TSDB_FUNC_PRJ, TSDB_BASE_FUNC_MO | TSDB_FUNCSTATE_NEED_TS, function_setup, - col_project_function, col_project_function_f, no_next_step, noop, copy_function, copy_function, + "ts", TSDB_FUNC_TS_COMP, TSDB_FUNC_TS_COMP, TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, ts_comp_function_setup, + ts_comp_function, ts_comp_function_f, no_next_step, ts_comp_finalize, copy_function, copy_function, data_req_load_info, }, { // 20 - "tagprj", TSDB_FUNC_TAGPRJ, TSDB_FUNC_TAGPRJ, - TSDB_BASE_FUNC_MO, // multi-output, tag function has only one result - function_setup, tag_project_function, tag_project_function_f, no_next_step, noop, copy_function, copy_function, - no_data_info, + "tag", TSDB_FUNC_TAG, TSDB_FUNC_TAG, TSDB_BASE_FUNC_SO, function_setup, tag_function, tag_function_f, + no_next_step, noop, copy_function, copy_function, no_data_info, }, { - // 21 - "arithmetic", TSDB_FUNC_ARITHM, TSDB_FUNC_ARITHM, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, function_setup, arithmetic_function, - arithmetic_function_f, no_next_step, noop, copy_function, copy_function, data_req_load_info, + // 21, column project sql function + "colprj", TSDB_FUNC_PRJ, TSDB_FUNC_PRJ, TSDB_BASE_FUNC_MO | TSDB_FUNCSTATE_NEED_TS, function_setup, + col_project_function, col_project_function_f, no_next_step, noop, copy_function, copy_function, + data_req_load_info, }, { - // 22 - "diff", TSDB_FUNC_DIFF, TSDB_FUNC_INVALID_ID, TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, diff_function_setup, - diff_function, diff_function_f, no_next_step, noop, noop, noop, data_req_load_info, + // 22, multi-output, tag function has only one result + "tagprj", TSDB_FUNC_TAGPRJ, TSDB_FUNC_TAGPRJ, TSDB_BASE_FUNC_MO, function_setup, tag_project_function, + tag_project_function_f, no_next_step, noop, copy_function, copy_function, no_data_info, }, - // distriubted version used in two-stage aggregation processes { // 23 - "sum_dst", TSDB_FUNC_SUM_DST, TSDB_FUNC_SUM_DST, TSDB_BASE_FUNC_SO, function_setup, sum_dist_intern_function, - sum_dist_intern_function_f, no_next_step, function_finalize, sum_dist_merge, sum_dist_second_merge, - precal_req_load_info, + "arithmetic", TSDB_FUNC_ARITHM, TSDB_FUNC_ARITHM, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, function_setup, arithmetic_function, + arithmetic_function_f, no_next_step, noop, copy_function, copy_function, data_req_load_info, }, { // 24 - "avg_dst", TSDB_FUNC_AVG_DST, TSDB_FUNC_AVG_DST, TSDB_BASE_FUNC_SO, avg_dist_function_setup, - avg_dist_intern_function, avg_dist_intern_function_f, no_next_step, avg_finalizer, avg_dist_merge, - avg_dist_second_merge, precal_req_load_info, + "diff", TSDB_FUNC_DIFF, TSDB_FUNC_INVALID_ID, TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, diff_function_setup, + diff_function, diff_function_f, no_next_step, noop, noop, noop, data_req_load_info, }, + // distributed version used in two-stage aggregation processes { // 25 - "min_dst", TSDB_FUNC_MIN_DST, TSDB_FUNC_MIN_DST, TSDB_BASE_FUNC_SO, min_function_setup, - min_dist_intern_function, min_dist_intern_function_f, no_next_step, function_finalize, min_dist_merge, - min_dist_second_merge, precal_req_load_info, + "first_dist", TSDB_FUNC_FIRST_DST, TSDB_FUNC_FIRST_DST, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, first_last_function_setup, + first_dist_function, first_dist_function_f, no_next_step, function_finalizer, first_dist_func_merge, + first_dist_func_second_merge, first_dist_data_req_info, }, { // 26 - "max_dst", TSDB_FUNC_MAX_DST, TSDB_FUNC_MAX_DST, TSDB_BASE_FUNC_SO, max_function_setup, - max_dist_intern_function, max_dist_intern_function_f, no_next_step, function_finalize, max_dist_merge, - max_dist_second_merge, precal_req_load_info, + "last_dist", TSDB_FUNC_LAST_DST, TSDB_FUNC_LAST_DST, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, first_last_function_setup, + last_dist_function, last_dist_function_f, no_next_step, function_finalizer, last_dist_func_merge, + last_dist_func_second_merge, last_dist_data_req_info, }, { // 27 - "first_dist", TSDB_FUNC_FIRST_DST, TSDB_FUNC_FIRST_DST, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - function_setup, first_dist_function, first_dist_function_f, no_next_step, function_finalize, first_dist_merge, - first_dist_second_merge, first_dist_data_req_info, - }, - { - // 28 - "last_dist", TSDB_FUNC_LAST_DST, TSDB_FUNC_LAST_DST, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, function_setup, - last_dist_function, last_dist_function_f, no_next_step, function_finalize, last_dist_merge, - last_dist_second_merge, last_dist_data_req_info, - }, - { - // 29 - "last_row_dist", TSDB_FUNC_LAST_ROW_DST, TSDB_FUNC_LAST_ROW_DST, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - function_setup, last_row_dist_function, noop, no_next_step, function_finalize, noop, last_dist_second_merge, - data_req_load_info, // this function is not necessary - }, - { - // 30 - "spread_dst", TSDB_FUNC_SPREAD_DST, TSDB_FUNC_SPREAD_DST, TSDB_BASE_FUNC_SO, spread_dist_function_setup, - spread_dist_intern_function, spread_dist_intern_function_f, no_next_step, - spread_function_finalize, // no finalize - spread_dist_merge, spread_dist_second_merge, count_load_data_info, - }, - { - // 31 - "wavg_dst", TSDB_FUNC_WAVG_DST, TSDB_FUNC_WAVG_DST, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, function_setup, - wavg_dist_function, wavg_dist_function_f, no_next_step, noop, wavg_dist_merge, wavg_dist_second_merge, - data_req_load_info, - }, - { - // 32 - "top_dst", TSDB_FUNC_TOP_DST, TSDB_FUNC_TOP_DST, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, top_bottom_dist_function_setup, - top_dist_function, top_dist_function_f, no_next_step, top_bottom_function_finalizer, top_dist_merge, - top_dist_second_merge, data_req_load_info, - }, - { - // 33 - "bottom_dst", TSDB_FUNC_BOTTOM_DST, TSDB_FUNC_BOTTOM_DST, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, top_bottom_dist_function_setup, - bottom_dist_function, bottom_dist_function_f, no_next_step, top_bottom_function_finalizer, bottom_dist_merge, - bottom_dist_second_merge, data_req_load_info, - }, - { - // 34 - "apercentile_dst", TSDB_FUNC_APERCT_DST, TSDB_FUNC_APERCT_DST, TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_METRIC, - apercentile_dist_function_setup, apercentile_dist_intern_function, apercentile_dist_intern_function_f, - no_next_step, apercentile_finalizer, apercentile_dist_merge, apercentile_dist_second_merge, data_req_load_info, - }, - { - // 35 "interp", TSDB_FUNC_INTERP, TSDB_FUNC_INTERP, TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, function_setup, interp_function, - sum_function_f, // todo filter handle + do_sum_f, // todo filter handle no_next_step, noop, noop, copy_function, no_data_info, }}; diff --git a/src/client/src/tscJoinProcess.c b/src/client/src/tscJoinProcess.c new file mode 100644 index 0000000000..4965498eff --- /dev/null +++ b/src/client/src/tscJoinProcess.c @@ -0,0 +1,1496 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include + +#include "os.h" +#include "tcache.h" +#include "tscJoinProcess.h" +#include "tscUtil.h" +#include "tsclient.h" +#include "tscompression.h" +#include "ttime.h" +#include "tutil.h" + +static UNUSED_FUNC bool isSubqueryCompleted(SSqlObj* pSql) { + bool hasData = true; + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + SSqlRes* pRes = &pSql->pSubs[i]->res; + + // in case inner join, if any subquery exhausted, query completed + if (pRes->numOfRows == 0) { + hasData = false; + break; + } + } + + return hasData; +} + +static bool doCompare(int32_t order, int64_t left, int64_t right) { + if (order == TSQL_SO_ASC) { + return left < right; + } else { + return left > right; + } +} + +static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSubquerySupporter* pSupporter1, SJoinSubquerySupporter* pSupporter2, + TSKEY* st, TSKEY* et) { + STSBuf* output1 = tsBufCreate(true); + STSBuf* output2 = tsBufCreate(true); + + *st = INT64_MAX; + *et = INT64_MIN; + + SLimitVal* pLimit = &pSql->cmd.limit; + int32_t order = pSql->cmd.order.order; + + pSql->pSubs[0]->cmd.tsBuf = output1; + pSql->pSubs[1]->cmd.tsBuf = output2; + + tsBufResetPos(pSupporter1->pTSBuf); + tsBufResetPos(pSupporter2->pTSBuf); + + // TODO add more details information + if (!tsBufNextPos(pSupporter1->pTSBuf)) { + tsBufFlush(output1); + tsBufFlush(output2); + + tscTrace("%p input1 is empty, 0 for secondary query after ts blocks intersecting", pSql); + return 0; + } + + if (!tsBufNextPos(pSupporter2->pTSBuf)) { + tsBufFlush(output1); + tsBufFlush(output2); + + tscTrace("%p input2 is empty, 0 for secondary query after ts blocks intersecting", pSql); + return 0; + } + + int64_t numOfInput1 = 1; + int64_t numOfInput2 = 1; + + while (1) { + STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf); + STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf); + +#ifdef _DEBUG_VIEW + // for debug purpose + tscPrint("%lld, tags:%d \t %lld, tags:%d", elem1.ts, elem1.tag, elem2.ts, elem2.tag); +#endif + + if (elem1.tag < elem2.tag || (elem1.tag == elem2.tag && doCompare(order, elem1.ts, elem2.ts))) { + if (!tsBufNextPos(pSupporter1->pTSBuf)) { + break; + } + + numOfInput1++; + } else if (elem1.tag > elem2.tag || (elem1.tag == elem2.tag && doCompare(order, elem2.ts, elem1.ts))) { + if (!tsBufNextPos(pSupporter2->pTSBuf)) { + break; + } + + numOfInput2++; + } else { + if (*st > elem1.ts) { + *st = elem1.ts; + } + + if (*et < elem1.ts) { + *et = elem1.ts; + } + + // in case of stable query, limit/offset is not applied here + if (pLimit->offset == 0 || pSql->cmd.nAggTimeInterval > 0 || QUERY_IS_STABLE_QUERY(pSql->cmd.type)) { + tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); + tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); + } else { + pLimit->offset -= 1; + } + + if (!tsBufNextPos(pSupporter1->pTSBuf)) { + break; + } + + numOfInput1++; + + if (!tsBufNextPos(pSupporter2->pTSBuf)) { + break; + } + + numOfInput2++; + } + } + + /* + * failed to set the correct ts order yet in two cases: + * 1. only one element + * 2. only one element for each tag. + */ + if (output1->tsOrder == -1) { + output1->tsOrder = TSQL_SO_ASC; + output2->tsOrder = TSQL_SO_ASC; + } + + tsBufFlush(output1); + tsBufFlush(output2); + + tsBufDestory(pSupporter1->pTSBuf); + tsBufDestory(pSupporter2->pTSBuf); + + tscTrace("%p input1:%lld, input2:%lld, %lld for secondary query after ts blocks intersecting", + pSql, numOfInput1, numOfInput2, output1->numOfTotal); + + return output1->numOfTotal; +} + +//todo handle failed to create sub query +SJoinSubquerySupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, /*int32_t* numOfComplete, int32_t* gc,*/ int32_t index) { + SJoinSubquerySupporter* pSupporter = calloc(1, sizeof(SJoinSubquerySupporter)); + if (pSupporter == NULL) { + return NULL; + } + + pSupporter->pObj = pSql; + pSupporter->hasMore = true; + + pSupporter->pState = pState; + + pSupporter->subqueryIndex = index; + pSupporter->interval = pSql->cmd.nAggTimeInterval; + pSupporter->limit = pSql->cmd.limit; + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, index); + pSupporter->uid = pMeterMetaInfo->pMeterMeta->uid; + + getTmpfilePath("join-", pSupporter->path); + pSupporter->f = fopen(pSupporter->path, "w"); + + if (pSupporter->f == NULL) { + tscError("%p failed to create tmp file:%s, reason:%s", pSql, pSupporter->path, strerror(errno)); + } + + return pSupporter; +} + +void tscDestroyJoinSupporter(SJoinSubquerySupporter* pSupporter) { + if (pSupporter == NULL) { + return; + } + + tfree(pSupporter->exprsInfo.pExprs); + tscColumnBaseInfoDestroy(&pSupporter->colList); + + tscClearFieldInfo(&pSupporter->fieldsInfo); + + if (pSupporter->f != NULL) { + fclose(pSupporter->f); + unlink(pSupporter->path); + } + + tscTagCondRelease(&pSupporter->tagCond); + free(pSupporter); +} + +/* + * need the secondary query process + * In case of count(ts)/count(*)/spread(ts) query, that are only applied to + * primary timestamp column , the secondary query is not necessary + * + */ +bool needSecondaryQuery(SSqlObj* pSql) { + SSqlCmd* pCmd = &pSql->cmd; + for (int32_t i = 0; i < pCmd->colList.numOfCols; ++i) { + SColumnBase* pBase = tscColumnBaseInfoGet(&pCmd->colList, i); + if (pBase->colIndex.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + return true; + } + } + + return false; +} + +/* + * launch secondary stage query to fetch the result that contains timestamp in set + */ +int32_t tscLaunchSecondSubquery(SSqlObj* pSql) { + // TODO not launch secondary stage query + // if (!needSecondaryQuery(pSql)) { + // return; + // } + + // sub query may not be necessary + int32_t numOfSub = 0; + SJoinSubquerySupporter* pSupporter = NULL; + + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + pSupporter = pSql->pSubs[i]->param; + pSupporter->pState->numOfCompleted = 0; + + if (pSupporter->exprsInfo.numOfExprs > 0) { + ++numOfSub; + } + } + + // scan all subquery, if one sub query has only ts, ignore it + int32_t j = 0; + tscTrace("%p start to launch secondary subqueries: %d", pSql, pSql->numOfSubs); + + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + SSqlObj* pSub = pSql->pSubs[i]; + pSupporter = pSub->param; + pSupporter->pState->numOfTotal = numOfSub; + + if (pSupporter->exprsInfo.numOfExprs == 0) { + tscDestroyJoinSupporter(pSupporter); + taos_free_result(pSub); + continue; + } + + SSqlObj* pNew = createSubqueryObj(pSql, 0, (int16_t)i, tscJoinQueryCallback, pSupporter, NULL); + if (pNew == NULL) { + pSql->numOfSubs = i; //revise the number of subquery + pSupporter->pState->numOfTotal = i; + + pSupporter->pState->code = TSDB_CODE_CLI_OUT_OF_MEMORY; + tscDestroyJoinSupporter(pSupporter); + + return NULL; + } + + tscFreeSqlCmdData(&pNew->cmd); + + pSql->pSubs[j++] = pNew; + pNew->cmd.tsBuf = pSub->cmd.tsBuf; + pSub->cmd.tsBuf = NULL; + + taos_free_result(pSub); + + // set the second stage sub query for join process + pNew->cmd.type |= TSDB_QUERY_TYPE_JOIN_SEC_STAGE; + + pNew->cmd.nAggTimeInterval = pSupporter->interval; + pNew->cmd.limit = pSupporter->limit; + pNew->cmd.groupbyExpr = pSupporter->groupbyExpr; + + tscColumnBaseInfoCopy(&pNew->cmd.colList, &pSupporter->colList, 0); + tscTagCondCopy(&pNew->cmd.tagCond, &pSupporter->tagCond); + + tscSqlExprCopy(&pNew->cmd.exprsInfo, &pSupporter->exprsInfo, pSupporter->uid); + tscFieldInfoCopyAll(&pSupporter->fieldsInfo, &pNew->cmd.fieldsInfo); + + // add the ts function for interval query if it is missing + if (pSupporter->exprsInfo.pExprs[0].functionId != TSDB_FUNC_TS && pNew->cmd.nAggTimeInterval > 0) { + tscAddTimestampColumn(&pNew->cmd, TSDB_FUNC_TS, 0); + } + + // todo refactor function name + tscAddTimestampColumn(&pNew->cmd, TSDB_FUNC_TS, 0); + tscFieldInfoCalOffset(&pNew->cmd); + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(&pNew->cmd, 0); + + // fetch the join tag column + if (UTIL_METER_IS_METRIC(pMeterMetaInfo)) { + SSqlExpr* pExpr = tscSqlExprGet(&pNew->cmd, 0); + assert(pNew->cmd.tagCond.joinInfo.hasJoin); + + int16_t tagColIndex = tscGetJoinTagColIndexByUid(&pNew->cmd, pMeterMetaInfo->pMeterMeta->uid); + pExpr->param[0].i64Key = tagColIndex; + pExpr->numOfParams = 1; + + addRequiredTagColumn(&pNew->cmd, tagColIndex, 0); + } + + tscProcessSql(pNew); + } + + // revise the number of subs + pSql->numOfSubs = j; + + return 0; +} + +static void freeSubqueryObj(SSqlObj* pSql) { + SSubqueryState* pState = NULL; + + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + if (pSql->pSubs[i] != NULL) { + SJoinSubquerySupporter* p = pSql->pSubs[i]->param; + pState = p->pState; + + tscDestroyJoinSupporter(p); + + if (pSql->pSubs[i]->res.code == TSDB_CODE_SUCCESS) { + taos_free_result(pSql->pSubs[i]); + } + } + } + + tfree(pState); + pSql->numOfSubs = 0; +} + +static void doQuitSubquery(SSqlObj* pParentSql) { + freeSubqueryObj(pParentSql); + + tsem_wait(&pParentSql->emptyRspSem); + tsem_wait(&pParentSql->emptyRspSem); + + tsem_post(&pParentSql->rspSem); +} + +static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSubquerySupporter* pSupporter) { + if (__sync_add_and_fetch_32(&pSupporter->pState->numOfCompleted, 1) >= pSupporter->pState->numOfTotal) { + pSqlObj->res.code = abs(pSupporter->pState->code); + tscError("%p all subquery return and query failed, global code:%d", pSqlObj, pSqlObj->res.code); + + doQuitSubquery(pSqlObj); + } +} + +// update the query time range according to the join results on timestamp +static void updateQueryTimeRange(SSqlObj* pSql, int64_t st, int64_t et) { + assert(pSql->cmd.stime <= st && pSql->cmd.etime >= et); + + pSql->cmd.stime = st; + pSql->cmd.etime = et; +} + +static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { + SJoinSubquerySupporter* pSupporter = (SJoinSubquerySupporter*)param; + SSqlObj* pParentSql = pSupporter->pObj; + + SSqlObj* pSql = (SSqlObj*)tres; + + if ((pSql->cmd.type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) == 0) { + if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { + tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, numOfRows, + pSupporter->pState->code); + + quitAllSubquery(pParentSql, pSupporter); + return; + } + + if (numOfRows > 0) { // write the data into disk + fwrite(pSql->res.data, pSql->res.numOfRows, 1, pSupporter->f); + fflush(pSupporter->f); + + STSBuf* pBuf = tsBufCreateFromFile(pSupporter->path, true); + if (pBuf == NULL) { + tscError("%p invalid ts comp file from vnode, abort sub query, file size:%d", pSql, numOfRows); + + pSupporter->pState->code = TSDB_CODE_APP_ERROR; // todo set the informative code + quitAllSubquery(pParentSql, pSupporter); + return; + } + + if (pSupporter->pTSBuf == NULL) { + tscTrace("%p create tmp file for ts block:%s", pSql, pBuf->path); + pSupporter->pTSBuf = pBuf; + } else { + tsBufMerge(pSupporter->pTSBuf, pBuf, pSql->cmd.vnodeIdx); + tsBufDestory(pBuf); + } + + // open new file to save the result + getTmpfilePath("ts-join", pSupporter->path); + pSupporter->f = fopen(pSupporter->path, "w"); + pSql->res.row = pSql->res.numOfRows; + + taos_fetch_rows_a(tres, joinRetrieveCallback, param); + } else if (numOfRows == 0) { // no data from this vnode anymore + if (__sync_add_and_fetch_32(&pSupporter->pState->numOfCompleted, 1) >= pSupporter->pState->numOfTotal) { + + if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { + tscTrace("%p sub:%p, numOfSub:%d, quit from further procedure due to other queries failure", pParentSql, tres, + pSupporter->subqueryIndex); + doQuitSubquery(pParentSql); + return; + } + + tscTrace("%p all subqueries retrieve ts complete, do ts block intersect", pParentSql); + + SJoinSubquerySupporter* p1 = pParentSql->pSubs[0]->param; + SJoinSubquerySupporter* p2 = pParentSql->pSubs[1]->param; + + TSKEY st, et; + + int64_t num = doTSBlockIntersect(pParentSql, p1, p2, &st, &et); + if (num <= 0) { // no result during ts intersect + tscTrace("%p free all sub SqlObj and quit", pParentSql); + doQuitSubquery(pParentSql); + } else { + updateQueryTimeRange(pParentSql, st, et); + tscLaunchSecondSubquery(pParentSql); + } + } + } else { // failure of sub query + tscError("%p sub query failed, code:%d, index:%d", pSql, numOfRows, pSupporter->subqueryIndex); + pSupporter->pState->code = numOfRows; + + quitAllSubquery(pParentSql, pSupporter); + return; + } + + } else { // secondary stage retrieve, driven by taos_fetch_row or other functions + if (numOfRows < 0) { + pSupporter->pState->code = numOfRows; + tscError("%p retrieve failed, code:%d, index:%d", pSql, numOfRows, pSupporter->subqueryIndex); + } + + if (__sync_add_and_fetch_32(&pSupporter->pState->numOfCompleted, 1) >= pSupporter->pState->numOfTotal) { + tscTrace("%p secondary retrieve completed, global code:%d", tres, pParentSql->res.code); + if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { + pParentSql->res.code = abs(pSupporter->pState->code); + freeSubqueryObj(pParentSql); + } + + tsem_post(&pParentSql->rspSem); + } + } +} + +void tscFetchDatablockFromSubquery(SSqlObj* pSql) { + int32_t numOfFetch = 0; + + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + SJoinSubquerySupporter* pSupporter = (SJoinSubquerySupporter*)pSql->pSubs[i]->param; + + SSqlRes* pRes = &pSql->pSubs[i]->res; + if (pRes->row >= pRes->numOfRows && pSupporter->hasMore) { + numOfFetch++; + } + } + + if (numOfFetch > 0) { + tscTrace("%p retrieve data from %d subqueries", pSql, numOfFetch); + + SJoinSubquerySupporter* pSupporter = (SJoinSubquerySupporter*)pSql->pSubs[0]->param; + pSupporter->pState->numOfTotal = numOfFetch; // wait for all subqueries completed + pSupporter->pState->numOfCompleted = 0; + + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + SSqlObj* pSql1 = pSql->pSubs[i]; + + SSqlRes* pRes1 = &pSql1->res; + SSqlCmd* pCmd1 = &pSql1->cmd; + + pSupporter = (SJoinSubquerySupporter*)pSql1->param; + + // wait for all subqueries completed + pSupporter->pState->numOfTotal = numOfFetch; + if (pRes1->row >= pRes1->numOfRows && pSupporter->hasMore) { + tscTrace("%p subquery:%p retrieve data from vnode, index:%d", pSql, pSql1, pSupporter->subqueryIndex); + + tscResetForNextRetrieve(pRes1); + + pSql1->fp = joinRetrieveCallback; + + if (pCmd1->command < TSDB_SQL_LOCAL) { + pCmd1->command = (pCmd1->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + } + + tscProcessSql(pSql1); + } + } + + // wait for all subquery completed + tsem_wait(&pSql->rspSem); + } +} + +// all subqueries return, set the result output index +void tscSetupOutputColumnIndex(SSqlObj* pSql) { + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + + tscTrace("%p all subquery response, retrieve data", pSql); + + pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * pCmd->fieldsInfo.numOfOutputCols); + + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); + + int32_t tableIndexOfSub = -1; + for (int32_t j = 0; j < pCmd->numOfTables; ++j) { + SSqlObj* pSub = pSql->pSubs[j]; + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(&pSub->cmd, 0); + if (pMeterMetaInfo->pMeterMeta->uid == pExpr->uid) { + tableIndexOfSub = j; + break; + } + } + + SSqlCmd* pSubCmd = &pSql->pSubs[tableIndexOfSub]->cmd; + + for (int32_t k = 0; k < pSubCmd->exprsInfo.numOfExprs; ++k) { + SSqlExpr* pSubExpr = tscSqlExprGet(pSubCmd, k); + if (pExpr->functionId == pSubExpr->functionId && pExpr->colInfo.colId == pSubExpr->colInfo.colId) { + pRes->pColumnIndex[i] = (SColumnIndex){.tableIndex = tableIndexOfSub, .columnIndex = k}; + break; + } + } + } +} + +void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { + SSqlObj* pSql = (SSqlObj*)tres; + // SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0); + + // int32_t idx = pSql->cmd.vnodeIdx; + + // SVnodeSidList *vnodeInfo = NULL; + // if (pMeterMetaInfo->pMetricMeta != NULL) { + // vnodeInfo = tscGetVnodeSidList(pMeterMetaInfo->pMetricMeta, idx - 1); + // } + + SJoinSubquerySupporter* pSupporter = (SJoinSubquerySupporter*)param; + + // if (__sync_add_and_fetch_32(pSupporter->numOfComplete, 1) >= + // pSupporter->numOfTotal) { + // SSqlObj *pParentObj = pSupporter->pObj; + // + // if ((pSql->cmd.type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) != 1) { + // int32_t num = 0; + // tscFetchDatablockFromSubquery(pParentObj); + // TSKEY* ts = tscGetQualifiedTSList(pParentObj, &num); + // + // if (num <= 0) { + // // no qualified result + // } + // + // tscLaunchSecondSubquery(pSql, ts, num); + // } else { + + // } + // } else { + if ((pSql->cmd.type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) != TSDB_QUERY_TYPE_JOIN_SEC_STAGE) { + if (code != TSDB_CODE_SUCCESS) { // direct call joinRetrieveCallback and set the error code + joinRetrieveCallback(param, pSql, code); + } else { // first stage query, continue to retrieve data + pSql->fp = joinRetrieveCallback; + pSql->cmd.command = TSDB_SQL_FETCH; + tscProcessSql(pSql); + } + + } else { // second stage join subquery + SSqlObj* pParentSql = pSupporter->pObj; + + if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { + tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, code, + pSupporter->pState->code); + quitAllSubquery(pParentSql, pSupporter); + + return; + } + + if (code != TSDB_CODE_SUCCESS) { + tscError("%p sub query failed, code:%d, set global code:%d, index:%d", pSql, code, code, + pSupporter->subqueryIndex); + pSupporter->pState->code = code; // todo set the informative code + + quitAllSubquery(pParentSql, pSupporter); + } else { + if (__sync_add_and_fetch_32(&pSupporter->pState->numOfCompleted, 1) >= pSupporter->pState->numOfTotal) { + tscSetupOutputColumnIndex(pParentSql); + + if (pParentSql->fp == NULL) { + tsem_wait(&pParentSql->emptyRspSem); + tsem_wait(&pParentSql->emptyRspSem); + + tsem_post(&pParentSql->rspSem); + } else { + // set the command flag must be after the semaphore been correctly set. + // pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; + // if (pPObj->res.code == TSDB_CODE_SUCCESS) { + // (*pPObj->fp)(pPObj->param, pPObj, 0); + // } else { + // tscQueueAsyncRes(pPObj); + // } + assert(0); + } + } + } + } +} + +static int32_t getDataStartOffset() { + return sizeof(STSBufFileHeader) + TS_COMP_FILE_VNODE_MAX * sizeof(STSVnodeBlockInfo); +} + +static int32_t doUpdateVnodeInfo(STSBuf* pTSBuf, int64_t offset, STSVnodeBlockInfo* pVInfo) { + if (offset < 0 || offset >= getDataStartOffset()) { + return -1; + } + + if (fseek(pTSBuf->f, offset, SEEK_SET) != 0) { + return -1; + } + + fwrite(pVInfo, sizeof(STSVnodeBlockInfo), 1, pTSBuf->f); + return 0; +} + +// update prev vnode length info in file +static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInfo* pBlockInfo) { + int32_t offset = sizeof(STSBufFileHeader) + index * sizeof(STSVnodeBlockInfo); + doUpdateVnodeInfo(pTSBuf, offset, pBlockInfo); +} + +static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { + const int32_t INITIAL_VNODEINFO_SIZE = 4; + + pTSBuf->numOfAlloc = INITIAL_VNODEINFO_SIZE; + pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSVnodeBlockInfoEx)); + if (pTSBuf->pData == NULL) { + tsBufDestory(pTSBuf); + return NULL; + } + + pTSBuf->tsData.rawBuf = malloc(MEM_BUF_SIZE); + if (pTSBuf->tsData.rawBuf == NULL) { + tsBufDestory(pTSBuf); + return NULL; + } + + pTSBuf->bufSize = MEM_BUF_SIZE; + pTSBuf->tsData.threshold = MEM_BUF_SIZE; + pTSBuf->tsData.allocSize = MEM_BUF_SIZE; + + pTSBuf->assistBuf = malloc(MEM_BUF_SIZE); + if (pTSBuf->assistBuf == NULL) { + tsBufDestory(pTSBuf); + return NULL; + } + + pTSBuf->block.payload = malloc(MEM_BUF_SIZE); + if (pTSBuf->block.payload == NULL) { + tsBufDestory(pTSBuf); + return NULL; + } + + pTSBuf->fileSize += getDataStartOffset(); + return pTSBuf; +} + +static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); + +/** + * todo error handling + * support auto closeable tmp file + * @param path + * @return + */ +STSBuf* tsBufCreate(bool autoDelete) { + STSBuf* pTSBuf = calloc(1, sizeof(STSBuf)); + if (pTSBuf == NULL) { + return NULL; + } + + getTmpfilePath("join", pTSBuf->path); + pTSBuf->f = fopen(pTSBuf->path, "w+"); + if (pTSBuf->f == NULL) { + free(pTSBuf); + return NULL; + } + + allocResForTSBuf(pTSBuf); + + // update the header info + STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = TSQL_SO_ASC}; + STSBufUpdateHeader(pTSBuf, &header); + + tsBufResetPos(pTSBuf); + pTSBuf->cur.order = TSQL_SO_ASC; + + pTSBuf->autoDelete = autoDelete; + pTSBuf->tsOrder = -1; + + return pTSBuf; +} + +STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { + STSBuf* pTSBuf = calloc(1, sizeof(STSBuf)); + if (pTSBuf == NULL) { + return NULL; + } + + strncpy(pTSBuf->path, path, PATH_MAX); + + pTSBuf->f = fopen(pTSBuf->path, "r"); + if (pTSBuf->f == NULL) { + return NULL; + } + + if (allocResForTSBuf(pTSBuf) == NULL) { + return NULL; + } + + // validate the file magic number + STSBufFileHeader header = {0}; + fseek(pTSBuf->f, 0, SEEK_SET); + fread(&header, 1, sizeof(header), pTSBuf->f); + + // invalid file + if (header.magic != TS_COMP_FILE_MAGIC) { + return NULL; + } + + if (header.numOfVnode > pTSBuf->numOfAlloc) { + pTSBuf->numOfAlloc = header.numOfVnode; + STSVnodeBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * pTSBuf->numOfAlloc); + if (tmp == NULL) { + tsBufDestory(pTSBuf); + return NULL; + } + + pTSBuf->pData = tmp; + } + + pTSBuf->numOfVnodes = header.numOfVnode; + + // check the ts order + pTSBuf->tsOrder = header.tsOrder; + if (pTSBuf->tsOrder != TSQL_SO_ASC && pTSBuf->tsOrder != TSQL_SO_DESC) { + tscError("invalid order info in buf:%d", pTSBuf->tsOrder); + tsBufDestory(pTSBuf); + return NULL; + } + + size_t infoSize = sizeof(STSVnodeBlockInfo) * pTSBuf->numOfVnodes; + + STSVnodeBlockInfo* buf = (STSVnodeBlockInfo*)calloc(1, infoSize); + int64_t pos = ftell(pTSBuf->f); + fread(buf, infoSize, 1, pTSBuf->f); + + // the length value for each vnode is not kept in file, so does not set the length value + for (int32_t i = 0; i < pTSBuf->numOfVnodes; ++i) { + STSVnodeBlockInfoEx* pBlockList = &pTSBuf->pData[i]; + memcpy(&pBlockList->info, &buf[i], sizeof(STSVnodeBlockInfo)); + } + + free(buf); + + fseek(pTSBuf->f, 0, SEEK_END); + + struct stat fileStat; + fstat(fileno(pTSBuf->f), &fileStat); + + pTSBuf->fileSize = (uint32_t) fileStat.st_size; + tsBufResetPos(pTSBuf); + + // ascending by default + pTSBuf->cur.order = TSQL_SO_ASC; + + pTSBuf->autoDelete = autoDelete; + return pTSBuf; +} + +void tsBufDestory(STSBuf* pTSBuf) { + if (pTSBuf == NULL) { + return; + } + + tfree(pTSBuf->assistBuf); + tfree(pTSBuf->tsData.rawBuf); + + tfree(pTSBuf->pData); + tfree(pTSBuf->block.payload); + + fclose(pTSBuf->f); + + if (pTSBuf->autoDelete) { + unlink(pTSBuf->path); + } + + free(pTSBuf); +} + +static STSVnodeBlockInfoEx* addOneVnodeInfo(STSBuf* pTSBuf, int32_t vnodeId) { + if (pTSBuf->numOfAlloc <= pTSBuf->numOfVnodes) { + uint32_t newSize = (uint32_t)(pTSBuf->numOfAlloc * 1.5); + assert(newSize > pTSBuf->numOfAlloc); + + STSVnodeBlockInfoEx* tmp = (STSVnodeBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * newSize); + if (tmp == NULL) { + return NULL; + } + + pTSBuf->pData = tmp; + pTSBuf->numOfAlloc = newSize; + memset(&pTSBuf->pData[pTSBuf->numOfVnodes], 0, sizeof(STSVnodeBlockInfoEx) * (newSize - pTSBuf->numOfVnodes)); + } + + if (pTSBuf->numOfVnodes > 0) { + STSVnodeBlockInfo* pPrevBlockInfo = &pTSBuf->pData[pTSBuf->numOfVnodes - 1].info; + + // update prev vnode length info in file + TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, pPrevBlockInfo); + } + + // set initial value for vnode block + STSVnodeBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfVnodes].info; + pBlockInfo->vnode = vnodeId; + pBlockInfo->offset = pTSBuf->fileSize; + assert(pBlockInfo->offset >= getDataStartOffset()); + + // update vnode info in file + TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes, pBlockInfo); + + // add one vnode info + pTSBuf->numOfVnodes += 1; + + // update the header info + STSBufFileHeader header = { + .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder}; + STSBufUpdateHeader(pTSBuf, &header); + + return &pTSBuf->pData[pTSBuf->numOfVnodes - 1]; +} + +static void shrinkBuffer(STSList* ptsData) { + // shrink tmp buffer size if it consumes too many memory compared to the pre-defined size + if (ptsData->allocSize >= ptsData->threshold * 2) { + ptsData->rawBuf = realloc(ptsData->rawBuf, MEM_BUF_SIZE); + ptsData->allocSize = MEM_BUF_SIZE; + } +} + +static void writeDataToDisk(STSBuf* pTSBuf) { + if (pTSBuf->tsData.len == 0) { + return; + } + + STSBlock* pBlock = &pTSBuf->block; + + pBlock->numOfElem = pTSBuf->tsData.len / TSDB_KEYSIZE; + pBlock->compLen = + tsCompressTimestamp(pTSBuf->tsData.rawBuf, pTSBuf->tsData.len, pTSBuf->tsData.len / TSDB_KEYSIZE, pBlock->payload, + pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); + + int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET); + UNUSED(r); + + /* + * format for output data: + * 1. tags, number of ts, size after compressed, payload, size after compressed + * 2. tags, number of ts, size after compressed, payload, size after compressed + * + * both side has the compressed length is used to support load data forwards/backwords. + */ + fwrite(&pBlock->tag, sizeof(pBlock->tag), 1, pTSBuf->f); + fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); + + fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); + + fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); + + fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); + + int32_t blockSize = sizeof(pBlock->tag) + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; + pTSBuf->fileSize += blockSize; + + pTSBuf->tsData.len = 0; + + pTSBuf->pData[pTSBuf->numOfVnodes - 1].info.compLen += blockSize; + pTSBuf->pData[pTSBuf->numOfVnodes - 1].info.numOfBlocks += 1; + + shrinkBuffer(&pTSBuf->tsData); +} + +static void expandBuffer(STSList* ptsData, int32_t inputSize) { + if (ptsData->allocSize - ptsData->len < inputSize) { + int32_t newSize = inputSize + ptsData->len; + char* tmp = realloc(ptsData->rawBuf, (size_t)newSize); + if (tmp == NULL) { + // todo + } + + ptsData->rawBuf = tmp; + ptsData->allocSize = newSize; + } +} + +STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { + STSBlock* pBlock = &pTSBuf->block; + + // clear the memory buffer + void* tmp = pBlock->payload; + memset(pBlock, 0, sizeof(STSBlock)); + pBlock->payload = tmp; + + if (order == TSQL_SO_DESC) { + /* + * set the right position for the reversed traverse, the reversed traverse is started from + * the end of each comp data block + */ + fseek(pTSBuf->f, -sizeof(pBlock->padding), SEEK_CUR); + fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); + + pBlock->compLen = pBlock->padding; + int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag); + fseek(pTSBuf->f, -offset, SEEK_CUR); + } + + fread(&pBlock->tag, sizeof(pBlock->tag), 1, pTSBuf->f); + fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); + + fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); + fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); + + if (decomp) { + pTSBuf->tsData.len = + tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, + pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); + } + + // read the comp length at the length of comp block + fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); + + // for backwards traverse, set the start position at the end of previous block + if (order == TSQL_SO_DESC) { + int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag); + int64_t r = fseek(pTSBuf->f, -offset, SEEK_CUR); + UNUSED(r); + } + + return pBlock; +} + +// set the order of ts buffer if the ts order has not been set yet +static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { + STSList* ptsData = &pTSBuf->tsData; + + if (pTSBuf->tsOrder == -1) { + if (ptsData->len > 0) { + TSKEY lastKey = *(TSKEY*)(ptsData->rawBuf + ptsData->len - TSDB_KEYSIZE); + + if (lastKey > *(TSKEY*)pData) { + pTSBuf->tsOrder = TSQL_SO_DESC; + } else { + pTSBuf->tsOrder = TSQL_SO_ASC; + } + } else if (len > TSDB_KEYSIZE) { + // no data in current vnode, more than one ts is added, check the orders + TSKEY k1 = *(TSKEY*)(pData); + TSKEY k2 = *(TSKEY*)(pData + TSDB_KEYSIZE); + + if (k1 < k2) { + pTSBuf->tsOrder = TSQL_SO_ASC; + } else if (k1 > k2) { + pTSBuf->tsOrder = TSQL_SO_DESC; + } else { + // todo handle error + } + } + } else { + // todo the timestamp order is set, check the asc/desc order of appended data + } + + return TSDB_CODE_SUCCESS; +} + +void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData, int32_t len) { + STSVnodeBlockInfoEx* pBlockInfo = NULL; + STSList* ptsData = &pTSBuf->tsData; + + if (pTSBuf->numOfVnodes == 0 || pTSBuf->pData[pTSBuf->numOfVnodes - 1].info.vnode != vnodeId) { + writeDataToDisk(pTSBuf); + shrinkBuffer(ptsData); + + pBlockInfo = addOneVnodeInfo(pTSBuf, vnodeId); + } else { + pBlockInfo = &pTSBuf->pData[pTSBuf->numOfVnodes - 1]; + } + + assert(pBlockInfo->info.vnode == vnodeId); + + if (pTSBuf->block.tag != tag && ptsData->len > 0) { + // new arrived data with different tags value, save current value into disk first + writeDataToDisk(pTSBuf); + } else { + expandBuffer(ptsData, len); + } + + pTSBuf->block.tag = tag; + memcpy(ptsData->rawBuf + ptsData->len, pData, (size_t)len); + + // todo check return value + setCheckTSOrder(pTSBuf, pData, len); + + ptsData->len += len; + pBlockInfo->len += len; + + pTSBuf->numOfTotal += len / TSDB_KEYSIZE; + + if (ptsData->len >= ptsData->threshold) { + writeDataToDisk(pTSBuf); + shrinkBuffer(ptsData); + } + + tsBufResetPos(pTSBuf); +} + +void tsBufFlush(STSBuf* pTSBuf) { + if (pTSBuf->tsData.len <= 0) { + return; + } + + writeDataToDisk(pTSBuf); + shrinkBuffer(&pTSBuf->tsData); + + STSVnodeBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfVnodes - 1].info; + + // update prev vnode length info in file + TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, pBlockInfo); + + // save the ts order into header + STSBufFileHeader header = { + .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder}; + STSBufUpdateHeader(pTSBuf, &header); + + fsync(fileno(pTSBuf->f)); +} + +static int32_t tsBufFindVnodeIndexFromId(STSVnodeBlockInfoEx* pVnodeInfoEx, int32_t numOfVnodes, int32_t vnodeId) { + int32_t j = -1; + for (int32_t i = 0; i < numOfVnodes; ++i) { + if (pVnodeInfoEx[i].info.vnode == vnodeId) { + j = i; + break; + } + } + + return j; +} + +// todo opt performance by cache blocks info +static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int32_t blockIndex) { + if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { + return -1; + } + + // sequentially read the compressed data blocks, start from the beginning of the comp data block of this vnode + int32_t i = 0; + bool decomp = false; + + while ((i++) <= blockIndex) { + if (readDataFromDisk(pTSBuf, TSQL_SO_ASC, decomp) == NULL) { + return -1; + } + } + + // set the file position to be the end of previous comp block + if (pTSBuf->cur.order == TSQL_SO_DESC) { + STSBlock* pBlock = &pTSBuf->block; + int32_t compBlockSize = + pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag); + fseek(pTSBuf->f, -compBlockSize, SEEK_CUR); + } + + return 0; +} + +static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int64_t tag) { + bool decomp = false; + + int64_t offset = 0; + if (pTSBuf->cur.order == TSQL_SO_ASC) { + offset = pBlockInfo->offset; + } else { // reversed traverse starts from the end of block + offset = pBlockInfo->offset + pBlockInfo->compLen; + } + + if (fseek(pTSBuf->f, offset, SEEK_SET) != 0) { + return -1; + } + + for (int32_t i = 0; i < pBlockInfo->numOfBlocks; ++i) { + if (readDataFromDisk(pTSBuf, pTSBuf->cur.order, decomp) == NULL) { + return -1; + } + + if (pTSBuf->block.tag == tag) { + return i; + } + } + + return -1; +} + +static void tsBufGetBlock(STSBuf* pTSBuf, int32_t vnodeIndex, int32_t blockIndex) { + STSVnodeBlockInfo* pBlockInfo = &pTSBuf->pData[vnodeIndex].info; + if (pBlockInfo->numOfBlocks <= blockIndex) { + assert(false); + } + + STSCursor* pCur = &pTSBuf->cur; + if (pCur->vnodeIndex == vnodeIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSQL_SO_ASC) || + (pCur->blockIndex >= blockIndex && pCur->order == TSQL_SO_DESC))) { + int32_t i = 0; + bool decomp = false; + int32_t step = abs(blockIndex - pCur->blockIndex); + + while ((++i) <= step) { + if (readDataFromDisk(pTSBuf, pCur->order, decomp) == NULL) { + return; + } + } + } else { + if (tsBufFindBlock(pTSBuf, pBlockInfo, blockIndex) == -1) { + assert(false); + } + } + + STSBlock* pBlock = &pTSBuf->block; + pTSBuf->tsData.len = + tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, + pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); + + assert(pTSBuf->tsData.len / TSDB_KEYSIZE == pBlock->numOfElem); + + pCur->vnodeIndex = vnodeIndex; + pCur->blockIndex = blockIndex; + + pCur->tsIndex = (pCur->order == TSQL_SO_ASC) ? 0 : pBlock->numOfElem - 1; +} + +STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId) { + int32_t j = tsBufFindVnodeIndexFromId(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId); + if (j == -1) { + return NULL; + } + + return &pTSBuf->pData[j].info; +} + +int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { + if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfVnode < 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { + return -1; + } + + int64_t r = fseek(pTSBuf->f, 0, SEEK_SET); + if (r != 0) { + return -1; + } + + fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f); + return 0; +} + +bool tsBufNextPos(STSBuf* pTSBuf) { + if (pTSBuf == NULL || pTSBuf->numOfVnodes == 0) { + return false; + } + + STSCursor* pCur = &pTSBuf->cur; + + // get the first/last position according to traverse order + if (pCur->vnodeIndex == -1) { + if (pCur->order == TSQL_SO_ASC) { + tsBufGetBlock(pTSBuf, 0, 0); + // list is empty + if (pTSBuf->block.numOfElem == 0) { + tsBufResetPos(pTSBuf); + return false; + } else { + return true; + } + } else { + int32_t vnodeIndex = pTSBuf->numOfVnodes - 1; + + int32_t vnodeId = pTSBuf->pData[pCur->vnodeIndex].info.vnode; + STSVnodeBlockInfo* pBlockInfo = tsBufGetVnodeBlockInfo(pTSBuf, vnodeId); + int32_t blockIndex = pBlockInfo->numOfBlocks - 1; + + tsBufGetBlock(pTSBuf, vnodeIndex, blockIndex); + + pCur->tsIndex = pTSBuf->block.numOfElem - 1; + if (pTSBuf->block.numOfElem == 0) { + tsBufResetPos(pTSBuf); + return false; + } else { + return true; + } + } + } + + int32_t step = pCur->order == TSQL_SO_ASC ? 1 : -1; + + while (1) { + assert(pTSBuf->tsData.len == pTSBuf->block.numOfElem * TSDB_KEYSIZE); + + if ((pCur->order == TSQL_SO_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) || + (pCur->order == TSQL_SO_DESC && pCur->tsIndex <= 0)) { + int32_t vnodeId = pTSBuf->pData[pCur->vnodeIndex].info.vnode; + + STSVnodeBlockInfo* pBlockInfo = tsBufGetVnodeBlockInfo(pTSBuf, vnodeId); + if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSQL_SO_ASC) || + (pCur->blockIndex <= 0 && pCur->order == TSQL_SO_DESC)) { + if ((pCur->vnodeIndex >= pTSBuf->numOfVnodes - 1 && pCur->order == TSQL_SO_ASC) || + (pCur->vnodeIndex <= 0 && pCur->order == TSQL_SO_DESC)) { + pCur->vnodeIndex = -1; + return false; + } + + int32_t blockIndex = pCur->order == TSQL_SO_ASC ? 0 : pBlockInfo->numOfBlocks - 1; + tsBufGetBlock(pTSBuf, pCur->vnodeIndex + step, blockIndex); + break; + + } else { + tsBufGetBlock(pTSBuf, pCur->vnodeIndex, pCur->blockIndex + step); + break; + } + } else { + pCur->tsIndex += step; + break; + } + } + + return true; +} + +void tsBufResetPos(STSBuf* pTSBuf) { + if (pTSBuf == NULL) { + return; + } + + pTSBuf->cur = (STSCursor){.tsIndex = -1, .blockIndex = -1, .vnodeIndex = -1, .order = pTSBuf->cur.order}; +} + +STSElem tsBufGetElem(STSBuf* pTSBuf) { + STSElem elem1 = {.vnode = -1}; + STSCursor* pCur = &pTSBuf->cur; + + if (pTSBuf == NULL || pCur->vnodeIndex < 0) { + return elem1; + } + + STSBlock* pBlock = &pTSBuf->block; + + elem1.vnode = pTSBuf->pData[pCur->vnodeIndex].info.vnode; + elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE); + elem1.tag = pBlock->tag; + + return elem1; +} + +/** + * current only support ts comp data from two vnode merge + * @param pDestBuf + * @param pSrcBuf + * @param vnodeId + * @return + */ +int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) { + if (pDestBuf == NULL || pSrcBuf == NULL || pSrcBuf->numOfVnodes <= 0) { + return 0; + } + + if (pDestBuf->numOfVnodes + pSrcBuf->numOfVnodes > TS_COMP_FILE_VNODE_MAX) { + return -1; + } + + // src can only have one vnode index + if (pSrcBuf->numOfVnodes > 1) { + return -1; + } + + // there are data in buffer, flush to disk first + tsBufFlush(pDestBuf); + + // compared with the last vnode id + if (vnodeId != pDestBuf->pData[pDestBuf->numOfVnodes - 1].info.vnode) { + int32_t oldSize = pDestBuf->numOfVnodes; + int32_t newSize = oldSize + pSrcBuf->numOfVnodes; + + if (pDestBuf->numOfAlloc < newSize) { + pDestBuf->numOfAlloc = newSize; + + STSVnodeBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSVnodeBlockInfoEx) * newSize); + if (tmp == NULL) { + return -1; + } + + pDestBuf->pData = tmp; + } + + // directly copy the vnode index information + memcpy(&pDestBuf->pData[oldSize], pSrcBuf->pData, (size_t)pSrcBuf->numOfVnodes * sizeof(STSVnodeBlockInfoEx)); + + // set the new offset value + for (int32_t i = 0; i < pSrcBuf->numOfVnodes; ++i) { + STSVnodeBlockInfoEx* pBlockInfoEx = &pDestBuf->pData[i + oldSize]; + pBlockInfoEx->info.offset = (pSrcBuf->pData[i].info.offset - getDataStartOffset()) + pDestBuf->fileSize; + pBlockInfoEx->info.vnode = vnodeId; + } + + pDestBuf->numOfVnodes = newSize; + } else { + STSVnodeBlockInfoEx* pBlockInfoEx = &pDestBuf->pData[pDestBuf->numOfVnodes - 1]; + pBlockInfoEx->len += pSrcBuf->pData[0].len; + pBlockInfoEx->info.numOfBlocks += pSrcBuf->pData[0].info.numOfBlocks; + pBlockInfoEx->info.compLen += pSrcBuf->pData[0].info.compLen; + pBlockInfoEx->info.vnode = vnodeId; + } + + int64_t r = fseek(pDestBuf->f, 0, SEEK_END); + assert(r == 0); + + int64_t offset = getDataStartOffset(); + int32_t size = pSrcBuf->fileSize - offset; + +#ifdef LINUX + ssize_t rc = sendfile(fileno(pDestBuf->f), fileno(pSrcBuf->f), &offset, size); +#else + ssize_t rc = fsendfile(pDestBuf->f, pSrcBuf->f, &offset, size); +#endif + if (rc == -1) { + printf("%s\n", strerror(errno)); + return -1; + } + + if (rc != size) { + printf("%s\n", strerror(errno)); + return -1; + } + + pDestBuf->numOfTotal += pSrcBuf->numOfTotal; + + return 0; +} + +STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order) { + STSBuf* pTSBuf = tsBufCreate(true); + + STSVnodeBlockInfo* pBlockInfo = &(addOneVnodeInfo(pTSBuf, 0)->info); + pBlockInfo->numOfBlocks = numOfBlocks; + pBlockInfo->compLen = len; + pBlockInfo->offset = getDataStartOffset(); + pBlockInfo->vnode = 0; + + // update prev vnode length info in file + TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, pBlockInfo); + + fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET); + fwrite((void*) pData, 1, len, pTSBuf->f); + pTSBuf->fileSize += len; + + pTSBuf->tsOrder = order; + assert(order == TSQL_SO_ASC || order == TSQL_SO_DESC); + + STSBufFileHeader header = { + .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder}; + STSBufUpdateHeader(pTSBuf, &header); + + fsync(fileno(pTSBuf->f)); + + return pTSBuf; +} + +STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag) { + STSElem elem = {.vnode = -1}; + + if (pTSBuf == NULL) { + return elem; + } + + int32_t j = tsBufFindVnodeIndexFromId(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId); + if (j == -1) { + return elem; + } + + // for debug purpose + // tsBufDisplay(pTSBuf); + + STSCursor* pCur = &pTSBuf->cur; + STSVnodeBlockInfo* pBlockInfo = &pTSBuf->pData[j].info; + + int32_t blockIndex = tsBufFindBlockByTag(pTSBuf, pBlockInfo, tag); + if (blockIndex < 0) { + return elem; + } + + pCur->vnodeIndex = j; + pCur->blockIndex = blockIndex; + tsBufGetBlock(pTSBuf, j, blockIndex); + + return tsBufGetElem(pTSBuf); +} + +STSCursor tsBufGetCursor(STSBuf* pTSBuf) { + STSCursor c = {.vnodeIndex = -1}; + if (pTSBuf == NULL) { + return c; + } + + return pTSBuf->cur; +} + +void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur) { + if (pTSBuf == NULL || pCur == NULL) { + return; + } + + // assert(pCur->vnodeIndex != -1 && pCur->tsIndex >= 0 && pCur->blockIndex >= 0); + if (pCur->vnodeIndex != -1) { + tsBufGetBlock(pTSBuf, pCur->vnodeIndex, pCur->blockIndex); + } + + pTSBuf->cur = *pCur; +} + +void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order) { + if (pTSBuf == NULL) { + return; + } + + pTSBuf->cur.order = order; +} + +STSBuf* tsBufClone(STSBuf* pTSBuf) { + if (pTSBuf == NULL) { + return NULL; + } + + return tsBufCreateFromFile(pTSBuf->path, false); +} + +void tsBufDisplay(STSBuf* pTSBuf) { + printf("-------start of ts comp file-------\n"); + printf("number of vnode:%d\n", pTSBuf->numOfVnodes); + + int32_t old = pTSBuf->cur.order; + pTSBuf->cur.order = TSQL_SO_ASC; + + tsBufResetPos(pTSBuf); + + while (tsBufNextPos(pTSBuf)) { + STSElem elem = tsBufGetElem(pTSBuf); + printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + pTSBuf->cur.order = old; + printf("-------end of ts comp file-------\n"); +} diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index b220c89741..9afb74fec2 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -16,8 +16,6 @@ #include #include #include -#include -#include #include "taosmsg.h" #include "tcache.h" @@ -40,11 +38,31 @@ static int32_t getToStringLength(char *pData, int32_t length, int32_t type) { return length; case TSDB_DATA_TYPE_NCHAR: return length; - case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_DOUBLE: { +#ifdef _TD_ARM_32_ + double dv = 0; + *(int64_t*)(&dv) = *(int64_t*)pData; + len = sprintf(buf, "%f", dv); +#else len = sprintf(buf, "%lf", *(double *)pData); +#endif + if (strncasecmp("nan", buf, 3) == 0) { + len = 4; + } + } break; - case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_FLOAT: { +#ifdef _TD_ARM_32_ + float fv = 0; + *(int32_t*)(&fv) = *(int32_t*)pData; + len = sprintf(buf, "%f", fv); +#else len = sprintf(buf, "%f", *(float *)pData); +#endif + if (strncasecmp("nan", buf, 3) == 0) { + len = 4; + } + } break; case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: @@ -69,9 +87,10 @@ static int32_t getToStringLength(char *pData, int32_t length, int32_t type) { * length((uint64_t) 123456789011) > 12, greater than sizsof(uint64_t) */ static int32_t tscMaxLengthOfTagsFields(SSqlObj *pSql) { - SMeterMeta *pMeta = pSql->cmd.pMeterMeta; + SMeterMeta *pMeta = tscGetMeterMetaInfo(&pSql->cmd, 0)->pMeterMeta; - if (pMeta->meterType != TSDB_METER_MTABLE) { + if (pMeta->meterType == TSDB_METER_METRIC || pMeta->meterType == TSDB_METER_OTABLE || + pMeta->meterType == TSDB_METER_STABLE) { return 0; } @@ -97,8 +116,9 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { SSqlRes *pRes = &pSql->res; // one column for each row - SSqlCmd * pCmd = &pSql->cmd; - SMeterMeta *pMeta = pCmd->pMeterMeta; + SSqlCmd * pCmd = &pSql->cmd; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + SMeterMeta * pMeta = pMeterMetaInfo->pMeterMeta; /* * tagValueCnt is to denote the number of tags columns for meter, not metric. and is to show the column data. @@ -109,7 +129,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { int32_t numOfRows = pMeta->numOfColumns; int32_t totalNumOfRows = numOfRows + pMeta->numOfTags; - if (UTIL_METER_IS_METRIC(pCmd)) { + if (UTIL_METER_IS_METRIC(pMeterMetaInfo)) { numOfRows = pMeta->numOfColumns + pMeta->numOfTags; } @@ -141,7 +161,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { } } - if (UTIL_METER_IS_METRIC(pCmd)) { + if (UTIL_METER_IS_METRIC(pMeterMetaInfo)) { return 0; } @@ -182,11 +202,25 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { case TSDB_DATA_TYPE_NCHAR: taosUcs4ToMbs(pTagValue, pSchema[i].bytes, target); break; - case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_FLOAT: { +#ifdef _TD_ARM_32_ + float fv = 0; + *(int32_t*)(&fv) = *(int32_t*)pTagValue; + sprintf(target, "%f", fv); +#else sprintf(target, "%f", *(float *)pTagValue); +#endif + } break; - case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_DOUBLE: { +#ifdef _TD_ARM_32_ + double dv = 0; + *(int64_t*)(&dv) = *(int64_t*)pTagValue; + sprintf(target, "%lf", dv); +#else sprintf(target, "%lf", *(double *)pTagValue); +#endif + } break; case TSDB_DATA_TYPE_TINYINT: sprintf(target, "%d", *(int8_t *)pTagValue); @@ -240,7 +274,7 @@ static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, } static int32_t tscProcessDescribeTable(SSqlObj *pSql) { - assert(pSql->cmd.pMeterMeta != NULL); + assert(tscGetMeterMetaInfo(&pSql->cmd, 0)->pMeterMeta != NULL); const int32_t NUM_OF_DESCRIBE_TABLE_COLUMNS = 4; const int32_t TYPE_COLUMN_LENGTH = 16; @@ -261,15 +295,17 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) { static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { // the result structure has been completed in sql parse, so we // only need to reorganize the results in the column format - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; + SSqlCmd * pCmd = &pSql->cmd; + SSqlRes * pRes = &pSql->res; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); - SMetricMeta *pMetricMeta = pCmd->pMetricMeta; - SSchema * pSchema = tsGetTagSchema(pCmd->pMeterMeta); + SMetricMeta *pMetricMeta = pMeterMetaInfo->pMetricMeta; + SSchema * pSchema = tsGetTagSchema(pMeterMetaInfo->pMeterMeta); int32_t vOffset[TSDB_MAX_COLUMNS] = {0}; - for (int32_t f = 1; f < pCmd->numOfReqTags; ++f) { - int16_t tagColumnIndex = pCmd->tagColumnIndex[f - 1]; + + for (int32_t f = 1; f < pMeterMetaInfo->numOfTags; ++f) { + int16_t tagColumnIndex = pMeterMetaInfo->tagColumnIndex[f - 1]; if (tagColumnIndex == -1) { vOffset[f] = vOffset[f - 1] + TSDB_METER_NAME_LEN; } else { @@ -290,10 +326,10 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { SMeterSidExtInfo *pSidExt = tscGetMeterSidInfo(pSidList, j); for (int32_t k = 0; k < pCmd->fieldsInfo.numOfOutputCols; ++k) { - SColIndex *pColIndex = &tscSqlExprGet(pCmd, k)->colInfo; - int32_t offsetId = pColIndex->colIdx; + SColIndexEx *pColIndex = &tscSqlExprGet(pCmd, k)->colInfo; + int16_t offsetId = pColIndex->colIdx; - assert(pColIndex->isTag); + assert((pColIndex->flag & TSDB_COL_TAG) != 0); char * val = pSidExt->tags + vOffset[offsetId]; TAOS_FIELD *pField = tscFieldInfoGetField(pCmd, k); @@ -312,7 +348,7 @@ static int tscBuildMetricTagSqlFunctionResult(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - SMetricMeta *pMetricMeta = pCmd->pMetricMeta; + SMetricMeta *pMetricMeta = tscGetMeterMetaInfo(pCmd, 0)->pMetricMeta; int32_t totalNumOfResults = 1; // count function only produce one result int32_t rowLen = tscGetResRowLength(pCmd); @@ -323,7 +359,7 @@ static int tscBuildMetricTagSqlFunctionResult(SSqlObj *pSql) { for (int32_t k = 0; k < pCmd->fieldsInfo.numOfOutputCols; ++k) { SSqlExpr *pExpr = tscSqlExprGet(pCmd, i); - if (pExpr->colInfo.colIdx == -1 && pExpr->sqlFuncId == TSDB_FUNC_COUNT) { + if (pExpr->colInfo.colIdx == -1 && pExpr->functionId == TSDB_FUNC_COUNT) { TAOS_FIELD *pField = tscFieldInfoGetField(pCmd, k); memcpy(pRes->data + tscFieldInfoGetOffset(pCmd, i) * totalNumOfResults + pField->bytes * rowIdx, @@ -342,7 +378,7 @@ static int tscBuildMetricTagSqlFunctionResult(SSqlObj *pSql) { static int tscProcessQueryTags(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; - SMeterMeta *pMeterMeta = pCmd->pMeterMeta; + SMeterMeta *pMeterMeta = tscGetMeterMetaInfo(pCmd, 0)->pMeterMeta; if (pMeterMeta == NULL || pMeterMeta->numOfTags == 0 || pMeterMeta->numOfColumns == 0) { strcpy(pCmd->payload, "invalid table"); pSql->res.code = TSDB_CODE_INVALID_TABLE; @@ -350,7 +386,7 @@ static int tscProcessQueryTags(SSqlObj *pSql) { } SSqlExpr *pExpr = tscSqlExprGet(pCmd, 0); - if (pExpr->sqlFuncId == TSDB_FUNC_COUNT) { + if (pExpr->functionId == TSDB_FUNC_COUNT) { return tscBuildMetricTagSqlFunctionResult(pSql); } else { return tscBuildMetricTagProjectionResult(pSql); @@ -367,7 +403,11 @@ int tscProcessLocalCmd(SSqlObj *pSql) { } else if (pCmd->command == TSDB_SQL_RETRIEVE_TAGS) { pSql->res.code = (uint8_t)tscProcessQueryTags(pSql); } else if (pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - pSql->res.qhandle = 0x1; // pass the qhandle check + /* + * pass the qhandle check, in order to call partial release function to + * free allocated resources and remove the SqlObj from linked list + */ + pSql->res.qhandle = 0x1; // pass the qhandle check pSql->res.numOfRows = 0; } else if (pCmd->command == TSDB_SQL_RESET_CACHE) { taosClearDataCache(tscCacheHandle); @@ -376,7 +416,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) { tscError("%p not support command:%d", pSql, pCmd->command); } - //keep the code in local variable in order to avoid invalid read in case of async query + // keep the code in local variable in order to avoid invalid read in case of async query int32_t code = pSql->res.code; if (pSql->fp != NULL) { // callback function diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 01f9f3f360..37a32e9ebb 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -21,14 +21,15 @@ #pragma GCC diagnostic ignored "-Woverflow" #pragma GCC diagnostic ignored "-Wunused-variable" +#include #include #include #include -#include #include #include +#include #include #include #include @@ -58,75 +59,68 @@ enum { TSDB_USE_CLI_TS = 1, }; -static void setErrMsg(char *msg, char *sql); +static void setErrMsg(char *msg, const char *sql); static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize); -// get formation -static int32_t getNumericType(const char *data) { - if (*data == '-' || *data == '+') { - data += 1; +static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) { + int32_t numType = isValidNumber(pToken); + if (TK_ILLEGAL == numType) { + return numType; } - if (data[0] == '0') { - if (data[1] == 'x' || data[1] == 'X') { - return TK_HEX; - } else { - return TK_OCT; - } - } else { - return TK_INTEGER; - } -} - -static int64_t tscToInteger(char *data, char **endPtr) { - int32_t numType = getNumericType(data); int32_t radix = 10; - if (numType == TK_HEX) { radix = 16; } else if (numType == TK_OCT) { radix = 8; + } else if (numType == TK_BIN) { + radix = 2; } - return strtoll(data, endPtr, radix); + *value = strtoll(pToken->z, endPtr, radix); + + return numType; } -int tsParseTime(char *value, int32_t valuelen, int64_t *time, char **next, char *error, int16_t timePrec) { - char * token; - int tokenlen; - int64_t interval; +static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) { + int32_t numType = isValidNumber(pToken); + if (TK_ILLEGAL == numType) { + return numType; + } + *value = strtod(pToken->z, endPtr); + return numType; +} - int64_t useconds = 0; +int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { + char * token; + int tokenlen; + int32_t index = 0; + SSQLToken sToken; + int64_t interval; + int64_t useconds = 0; + char * pTokenEnd = *next; - char *pTokenEnd = *next; + index = 0; - if (valuelen == 3 && (strncmp(value, "now", 3) == 0)) { + if (pToken->type == TK_NOW) { useconds = taosGetTimestamp(timePrec); - } else if (valuelen == 1 && value[0] == '0') { + } else if (strncmp(pToken->z, "0", 1) == 0 && pToken->n == 1) { // do nothing - } else if (valuelen <= 4 || value[4] != '-') { - for (int32_t i = 0; i < valuelen; ++i) { - /* - * filter illegal input. - * e.g., nw, tt, ff etc. - */ - if (value[i] < '0' || value[i] > '9') { - return TSDB_CODE_INVALID_SQL; - } - } - useconds = str2int64(value); + } else if (pToken->type == TK_INTEGER) { + useconds = str2int64(pToken->z); } else { // strptime("2001-11-12 18:31:01", "%Y-%m-%d %H:%M:%S", &tm); - if (taosParseTime(value, time, valuelen, timePrec) != TSDB_CODE_SUCCESS) { + if (taosParseTime(pToken->z, time, pToken->n, timePrec) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } return TSDB_CODE_SUCCESS; } - for (int k = valuelen; value[k] != '\0'; k++) { - if (value[k] == ' ' || value[k] == '\t') continue; - if (value[k] == ',') { + for (int k = pToken->n; pToken->z[k] != '\0'; k++) { + if (pToken->z[k] == ' ' || pToken->z[k] == '\t') continue; + if (pToken->z[k] == ',') { + *next = pTokenEnd; *time = useconds; return 0; } @@ -138,22 +132,27 @@ int tsParseTime(char *value, int32_t valuelen, int64_t *time, char **next, char * time expression: * e.g., now+12a, now-5h */ - pTokenEnd = tscGetToken(pTokenEnd, &token, &tokenlen); - if (tokenlen && (*token == '+' || *token == '-')) { - pTokenEnd = tscGetToken(pTokenEnd, &value, &valuelen); - if (valuelen < 2) { + SSQLToken valueToken; + index = 0; + sToken = tStrGetToken(pTokenEnd, &index, false, 0, NULL); + pTokenEnd += index; + if (sToken.type == TK_MINUS || sToken.type == TK_PLUS) { + index = 0; + valueToken = tStrGetToken(pTokenEnd, &index, false, 0, NULL); + pTokenEnd += index; + if (valueToken.n < 2) { strcpy(error, "value is expected"); return TSDB_CODE_INVALID_SQL; } - if (getTimestampInUsFromStr(value, valuelen, &interval) != TSDB_CODE_SUCCESS) { + if (getTimestampInUsFromStr(valueToken.z, valueToken.n, &interval) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } if (timePrec == TSDB_TIME_PRECISION_MILLI) { interval /= 1000; } - if (*token == '+') { + if (sToken.type == TK_PLUS) { useconds += interval; } else { useconds = (useconds >= interval) ? useconds - interval : 0; @@ -166,192 +165,181 @@ int tsParseTime(char *value, int32_t valuelen, int64_t *time, char **next, char return TSDB_CODE_SUCCESS; } -/* - * The server time/client time should not be mixed up in one sql string - * Do not employ sort operation is not involved if server time is used. - */ -static int32_t tsCheckTimestamp(STableDataBlocks* pDataBlocks, const char* start) { - // once the data block is disordered, we do NOT keep previous timestamp any more - if (!pDataBlocks->ordered) { - return TSDB_CODE_SUCCESS; - } - - TSKEY k = *(TSKEY *) start; - - if (k == 0) { - if (pDataBlocks->tsSource == TSDB_USE_CLI_TS) { - return -1; - } else if (pDataBlocks->tsSource == -1) { - pDataBlocks->tsSource = TSDB_USE_SERVER_TS ; - } - } else { - if (pDataBlocks->tsSource == TSDB_USE_SERVER_TS) { - return -1; - } else if (pDataBlocks->tsSource == -1) { - pDataBlocks->tsSource = TSDB_USE_CLI_TS; - } - } - - if (k <= pDataBlocks->prevTS && (pDataBlocks->tsSource == TSDB_USE_CLI_TS)) { - pDataBlocks->ordered = false; - } - - pDataBlocks->prevTS = k; - return TSDB_CODE_SUCCESS; -} - -int32_t tsParseOneColumnData(SSchema *pSchema, char *value, int valuelen, char *payload, char *msg, char **str, - bool primaryKey, int16_t timePrec) { - int64_t temp; - int32_t nullInt = *(int32_t *)TSDB_DATA_NULL_STR_L; +int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, char *msg, char **str, bool primaryKey, + int16_t timePrec) { + int64_t iv; + int32_t numType; char * endptr = NULL; errno = 0; // reset global error code switch (pSchema->type) { case TSDB_DATA_TYPE_BOOL: { // bool - if (valuelen == 4 && nullInt == *(int32_t *)value) { - *(uint8_t *)payload = TSDB_DATA_BOOL_NULL; - } else { - if (strncmp(value, "true", valuelen) == 0) { + if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { + if (strncmp(pToken->z, "true", pToken->n) == 0) { *(uint8_t *)payload = TSDB_TRUE; - } else if (strncmp(value, "false", valuelen) == 0) { + } else if (strncmp(pToken->z, "false", pToken->n) == 0) { *(uint8_t *)payload = TSDB_FALSE; + } else if (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0) { + *(uint8_t *)payload = TSDB_DATA_BOOL_NULL; } else { - int64_t v = strtoll(value, NULL, 10); - *(uint8_t *)payload = (int8_t)((v == 0) ? TSDB_FALSE : TSDB_TRUE); + INVALID_SQL_RET_MSG(msg, "data is illegal"); } + } else if (pToken->type == TK_INTEGER) { + iv = strtoll(pToken->z, NULL, 10); + *(uint8_t *)payload = (int8_t)((iv == 0) ? TSDB_FALSE : TSDB_TRUE); + } else if (pToken->type == TK_FLOAT) { + double dv = strtod(pToken->z, NULL); + *(uint8_t *)payload = (int8_t)((dv == 0) ? TSDB_FALSE : TSDB_TRUE); + } else if (pToken->type == TK_NULL) { + *(uint8_t *)payload = TSDB_DATA_BOOL_NULL; + } else { + INVALID_SQL_RET_MSG(msg, "data is illegal"); } break; } case TSDB_DATA_TYPE_TINYINT: - if (valuelen == 4 && nullInt == *(int32_t *)value) { - *((int32_t *)payload) = TSDB_DATA_TINYINT_NULL; + if (pToken->type == TK_NULL) { + *((int8_t *)payload) = TSDB_DATA_TINYINT_NULL; + } else if ((pToken->type == TK_STRING) && (pToken->n != 0) && + (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)) { + *((int8_t *)payload) = TSDB_DATA_TINYINT_NULL; } else { - int64_t v = tscToInteger(value, &endptr); - if (errno == ERANGE || v > INT8_MAX || v < INT8_MIN) { - INVALID_SQL_RET_MSG(msg, "data is overflow"); - } - - int8_t v8 = (int8_t)v; - if (isNull((char *)&v8, pSchema->type)) { + numType = tscToInteger(pToken, &iv, &endptr); + if (errno == ERANGE || iv > INT8_MAX || iv <= INT8_MIN) { INVALID_SQL_RET_MSG(msg, "data is overflow"); + } else if (TK_ILLEGAL == numType) { + INVALID_SQL_RET_MSG(msg, "data is illegal"); } - *((int8_t *)payload) = v8; + *((int8_t *)payload) = (int8_t)iv; } break; case TSDB_DATA_TYPE_SMALLINT: - if (valuelen == 4 && nullInt == *(int32_t *)value) { - *((int32_t *)payload) = TSDB_DATA_SMALLINT_NULL; + if (pToken->type == TK_NULL) { + *((int16_t *)payload) = TSDB_DATA_SMALLINT_NULL; + } else if ((pToken->type == TK_STRING) && (pToken->n != 0) && + (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)) { + *((int16_t *)payload) = TSDB_DATA_SMALLINT_NULL; } else { - int64_t v = tscToInteger(value, &endptr); - - if (errno == ERANGE || v > INT16_MAX || v < INT16_MIN) { - INVALID_SQL_RET_MSG(msg, "data is overflow"); - } - - int16_t v16 = (int16_t)v; - if (isNull((char *)&v16, pSchema->type)) { + numType = tscToInteger(pToken, &iv, &endptr); + if (errno == ERANGE || iv > INT16_MAX || iv <= INT16_MIN) { INVALID_SQL_RET_MSG(msg, "data is overflow"); + } else if (TK_ILLEGAL == numType) { + INVALID_SQL_RET_MSG(msg, "data is illegal"); } - *((int16_t *)payload) = v16; + *((int16_t *)payload) = (int16_t)iv; } break; case TSDB_DATA_TYPE_INT: - if (valuelen == 4 && nullInt == *(int32_t *)value) { + if (pToken->type == TK_NULL) { + *((int32_t *)payload) = TSDB_DATA_INT_NULL; + } else if ((pToken->type == TK_STRING) && (pToken->n != 0) && + (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)) { *((int32_t *)payload) = TSDB_DATA_INT_NULL; } else { - int64_t v = tscToInteger(value, &endptr); - - if (errno == ERANGE || v > INT32_MAX || v < INT32_MIN) { + numType = tscToInteger(pToken, &iv, &endptr); + if (errno == ERANGE || iv > INT32_MAX || iv <= INT32_MIN) { INVALID_SQL_RET_MSG(msg, "data is overflow"); + } else if (TK_ILLEGAL == numType) { + INVALID_SQL_RET_MSG(msg, "data is illegal"); } - int32_t v32 = (int32_t)v; - if (isNull((char *)&v32, pSchema->type)) { - INVALID_SQL_RET_MSG(msg, "data is overflow"); - } - - *((int32_t *)payload) = v32; + *((int32_t *)payload) = (int32_t)iv; } break; case TSDB_DATA_TYPE_BIGINT: - if (valuelen == 4 && nullInt == *(int32_t *)value) { + if (pToken->type == TK_NULL) { + *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; + } else if ((pToken->type == TK_STRING) && (pToken->n != 0) && + (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)) { *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; } else { - int64_t v = tscToInteger(value, &endptr); - if (isNull((char *)&v, pSchema->type) || errno == ERANGE) { + numType = tscToInteger(pToken, &iv, &endptr); + if (errno == ERANGE || iv > INT64_MAX || iv <= INT64_MIN) { INVALID_SQL_RET_MSG(msg, "data is overflow"); + } else if (TK_ILLEGAL == numType) { + INVALID_SQL_RET_MSG(msg, "data is illegal"); } - *((int64_t *)payload) = v; + + *((int64_t *)payload) = iv; } break; case TSDB_DATA_TYPE_FLOAT: - if (valuelen == 4 && nullInt == *(int32_t *)value) { + if (pToken->type == TK_NULL) { + *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; + } else if ((pToken->type == TK_STRING) && (pToken->n != 0) && + (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)) { *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; } else { - float v = (float)strtod(value, &endptr); - if (isNull((char *)&v, pSchema->type) || isinf(v) || isnan(v)) { - *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; - } else { - *((float *)payload) = v; + double dv; + if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { + INVALID_SQL_RET_MSG(msg, "data is illegal"); + } + + float fv = (float)dv; + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || (fv > FLT_MAX || fv < -FLT_MAX)) { + INVALID_SQL_RET_MSG(msg, "data is illegal"); } - if (str != NULL) { - // This if statement is just for Fanuc case, when a float point number is quoted by - // quotes, we need to skip the quote. But this is temporary, it should be changed in the future. - if (*endptr == '\'' || *endptr == '\"') endptr++; - *str = endptr; + if (isinf(fv) || isnan(fv)) { + *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; } + + *((float *)payload) = fv; } break; case TSDB_DATA_TYPE_DOUBLE: - if (valuelen == 4 && nullInt == *(int32_t *)value) { + if (pToken->type == TK_NULL) { + *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; + } else if ((pToken->type == TK_STRING) && (pToken->n != 0) && + (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)) { *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; } else { - double v = strtod(value, &endptr); - if (isNull((char *)&v, pSchema->type) || isinf(v) || isnan(v)) { - *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; - } else { - *((double *)payload) = v; + double dv; + if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { + INVALID_SQL_RET_MSG(msg, "data is illegal"); } - if (str != NULL) { - // This if statement is just for Fanuc case, when a float point number is quoted by - // quotes, we need to skip the quote. But this is temporary, it should be changed in the future. - if (*endptr == '\'' || *endptr == '\"') endptr++; - *str = endptr; + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || (dv > DBL_MAX || dv < -DBL_MAX)) { + INVALID_SQL_RET_MSG(msg, "data is illegal"); + } + + if (isinf(dv) || isnan(dv)) { + *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; + } else { + *((double *)payload) = dv; } } break; case TSDB_DATA_TYPE_BINARY: - /* - * binary data cannot be null-terminated char string, otherwise the last char of the string is lost - */ - if (valuelen == 4 && nullInt == *(int32_t *)value) { + // binary data cannot be null-terminated char string, otherwise the last char of the string is lost + if (pToken->type == TK_NULL) { *payload = TSDB_DATA_BINARY_NULL; } else { - /* truncate too long string */ - if (valuelen > pSchema->bytes) valuelen = pSchema->bytes; - strncpy(payload, value, valuelen); + // too long values will return invalid sql, not be truncated automatically + if (pToken->n > pSchema->bytes) { + INVALID_SQL_RET_MSG(msg, "value too long"); + } + strncpy(payload, pToken->z, pToken->n); } break; case TSDB_DATA_TYPE_NCHAR: - if (valuelen == 4 && nullInt == *(int32_t *)value) { + if (pToken->type == TK_NULL) { *(uint32_t *)payload = TSDB_DATA_NCHAR_NULL; } else { - if (!taosMbsToUcs4(value, valuelen, payload, pSchema->bytes)) { + // if the converted output len is over than pSchema->bytes, return error: 'Argument list too long' + if (!taosMbsToUcs4(pToken->z, pToken->n, payload, pSchema->bytes)) { sprintf(msg, "%s", strerror(errno)); return TSDB_CODE_INVALID_SQL; } @@ -359,14 +347,15 @@ int32_t tsParseOneColumnData(SSchema *pSchema, char *value, int valuelen, char * break; case TSDB_DATA_TYPE_TIMESTAMP: { - if (valuelen == 4 && nullInt == *(int32_t *)value) { + if (pToken->type == TK_NULL) { if (primaryKey) { *((int64_t *)payload) = 0; } else { *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; } } else { - if (tsParseTime(value, valuelen, &temp, str, msg, timePrec) != TSDB_CODE_SUCCESS) { + int64_t temp; + if (tsParseTime(pToken, &temp, str, msg, timePrec) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } *((int64_t *)payload) = temp; @@ -380,8 +369,8 @@ int32_t tsParseOneColumnData(SSchema *pSchema, char *value, int valuelen, char * } // todo merge the error msg function with tSQLParser -static void setErrMsg(char *msg, char *sql) { - const char* msgFormat = "near \"%s\" syntax error"; +static void setErrMsg(char *msg, const char *sql) { + const char * msgFormat = "near \"%s\" syntax error"; const int32_t BACKWARD_CHAR_STEP = 15; // only extract part of sql string,avoid too long sql string cause stack over flow @@ -390,44 +379,84 @@ static void setErrMsg(char *msg, char *sql) { sprintf(msg, msgFormat, buf); } +/* + * The server time/client time should not be mixed up in one sql string + * Do not employ sort operation is not involved if server time is used. + */ +static int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start) { + // once the data block is disordered, we do NOT keep previous timestamp any more + if (!pDataBlocks->ordered) { + return TSDB_CODE_SUCCESS; + } + + TSKEY k = *(TSKEY *)start; + + if (k == 0) { + if (pDataBlocks->tsSource == TSDB_USE_CLI_TS) { + return -1; + } else if (pDataBlocks->tsSource == -1) { + pDataBlocks->tsSource = TSDB_USE_SERVER_TS; + } + } else { + if (pDataBlocks->tsSource == TSDB_USE_SERVER_TS) { + return -1; + } else if (pDataBlocks->tsSource == -1) { + pDataBlocks->tsSource = TSDB_USE_CLI_TS; + } + } + + if (k <= pDataBlocks->prevTS && (pDataBlocks->tsSource == TSDB_USE_CLI_TS)) { + pDataBlocks->ordered = false; + } + + pDataBlocks->prevTS = k; + return TSDB_CODE_SUCCESS; +} + int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[], SParsedDataColInfo *spd, char *error, int16_t timePrec) { - char *value = NULL; - int valuelen = 0; - - char *payload = pDataBlocks->pData + pDataBlocks->size; + int32_t index = 0; + bool isPrevOptr; + SSQLToken sToken; + char * payload = pDataBlocks->pData + pDataBlocks->size; - /* 1. set the parsed value from sql string */ + // 1. set the parsed value from sql string int32_t rowSize = 0; for (int i = 0; i < spd->numOfAssignedCols; ++i) { - /* the start position in data block buffer of current value in sql */ + // the start position in data block buffer of current value in sql char * start = payload + spd->elems[i].offset; int16_t colIndex = spd->elems[i].colIndex; - rowSize += schema[colIndex].bytes; + SSchema *pSchema = schema + colIndex; + rowSize += pSchema->bytes; - int sign = 0; - _again: - *str = tscGetToken(*str, &value, &valuelen); - if ((valuelen == 0 && value == NULL) || (valuelen == 1 && value[0] == ')')) { - setErrMsg(error, *str); + index = 0; + sToken = tStrGetToken(*str, &index, true, 0, NULL); + *str += index; + + if (sToken.type == TK_QUESTION) { + uint32_t offset = start - pDataBlocks->pData; + if (tscAddParamToDataBlock(pDataBlocks, pSchema->type, (uint8_t)timePrec, pSchema->bytes, offset) != NULL) { + continue; + } + strcpy(error, "client out of memory"); return -1; } - /* support positive/negative integer/float data format */ - if ((*value == '+' || *value == '-') && valuelen == 1) { - sign = 1; - goto _again; + if (((sToken.type != TK_NOW) && (sToken.type != TK_INTEGER) && (sToken.type != TK_STRING) && + (sToken.type != TK_FLOAT) && (sToken.type != TK_BOOL) && (sToken.type != TK_NULL)) || + (sToken.n == 0) || (sToken.type == TK_RP)) { + setErrMsg(error, *str); + return -1; } - if (sign) { - value = value - 1; - /* backward to include the +/- symbol */ - valuelen++; + // Remove quotation marks + if (TK_STRING == sToken.type) { + sToken.z++; + sToken.n -= 2; } - bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); - int32_t ret = tsParseOneColumnData(&schema[colIndex], value, valuelen, start, error, str, - colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX, timePrec); + bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); + int32_t ret = tsParseOneColumnData(pSchema, &sToken, start, error, str, isPrimaryKey, timePrec); if (ret != TSDB_CODE_SUCCESS) { return -1; // NOTE: here 0 mean error! } @@ -437,13 +466,13 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[ } } - /*2. set the null value for the rest columns */ + // 2. set the null value for the columns that do not assign values if (spd->numOfAssignedCols < spd->numOfCols) { char *ptr = payload; for (int32_t i = 0; i < spd->numOfCols; ++i) { if (!spd->hasVal[i]) { - /* current column do not have any value to insert, set it to null */ + // current column do not have any value to insert, set it to null setNull(ptr, schema[i].type, schema[i].bytes); } @@ -469,8 +498,8 @@ static int32_t rowDataCompar(const void *lhs, const void *rhs) { int tsParseValues(char **str, STableDataBlocks *pDataBlock, SMeterMeta *pMeterMeta, int maxRows, SParsedDataColInfo *spd, char *error) { - char *token; - int tokenlen; + int32_t index = 0; + SSQLToken sToken; int16_t numOfRows = 0; @@ -478,17 +507,23 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, SMeterMeta *pMeterMe int32_t precision = pMeterMeta->precision; if (spd->hasVal[0] == false) { - sprintf(error, "primary timestamp column can not be null"); + strcpy(error, "primary timestamp column can not be null"); return -1; } while (1) { - char *tmp = tscGetToken(*str, &token, &tokenlen); - if (tokenlen == 0 || *token != '(') break; + index = 0; + sToken = tStrGetToken(*str, &index, false, 0, NULL); + if (sToken.n == 0 || sToken.type != TK_LP) break; - *str = tmp; + *str += index; if (numOfRows >= maxRows || pDataBlock->size + pMeterMeta->rowSize >= pDataBlock->nAllocSize) { - maxRows += tscAllocateMemIfNeed(pDataBlock, pMeterMeta->rowSize); + int32_t tSize = tscAllocateMemIfNeed(pDataBlock, pMeterMeta->rowSize); + if (0 == tSize) { + strcpy(error, "client out of memory"); + return -1; + } + maxRows += tSize; } int32_t len = tsParseOneRowData(str, pDataBlock, pSchema, spd, error, precision); @@ -499,8 +534,10 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, SMeterMeta *pMeterMe pDataBlock->size += len; - *str = tscGetToken(*str, &token, &tokenlen); - if (tokenlen == 0 || *token != ')') { + index = 0; + sToken = tStrGetToken(*str, &index, false, 0, NULL); + *str += index; + if (sToken.n == 0 || sToken.type != TK_RP) { setErrMsg(error, *str); return -1; } @@ -516,18 +553,6 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, SMeterMeta *pMeterMe } } -void tscAppendDataBlock(SDataBlockList *pList, STableDataBlocks *pBlocks) { - if (pList->nSize >= pList->nAlloc) { - pList->nAlloc = pList->nAlloc << 1; - pList->pData = realloc(pList->pData, sizeof(void *) * (size_t)pList->nAlloc); - - // reset allocated memory - memset(pList->pData + pList->nSize, 0, sizeof(void *) * (pList->nAlloc - pList->nSize)); - } - - pList->pData[pList->nSize++] = pBlocks; -} - static void tscSetAssignedColumnInfo(SParsedDataColInfo *spd, SSchema *pSchema, int32_t numOfCols) { spd->numOfCols = numOfCols; spd->numOfAssignedCols = numOfCols; @@ -543,13 +568,14 @@ static void tscSetAssignedColumnInfo(SParsedDataColInfo *spd, SSchema *pSchema, } int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize) { - size_t remain = pDataBlock->nAllocSize - pDataBlock->size; + size_t remain = pDataBlock->nAllocSize - pDataBlock->size; const int factor = 5; + uint32_t nAllocSizeOld = pDataBlock->nAllocSize; // expand the allocated size if (remain < rowSize * factor) { while (remain < rowSize * factor) { - pDataBlock->nAllocSize = (uint32_t) (pDataBlock->nAllocSize * 1.5); + pDataBlock->nAllocSize = (uint32_t)(pDataBlock->nAllocSize * 1.5); remain = pDataBlock->nAllocSize - pDataBlock->size; } @@ -558,8 +584,10 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize) { pDataBlock->pData = tmp; memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); } else { - assert(false); + //assert(false); // do nothing + pDataBlock->nAllocSize = nAllocSizeOld; + return 0; } } @@ -575,7 +603,7 @@ static void tsSetBlockInfo(SShellSubmitBlock *pBlocks, const SMeterMeta *pMeterM // data block is disordered, sort it in ascending order void sortRemoveDuplicates(STableDataBlocks *dataBuf) { - SShellSubmitBlock* pBlocks = (SShellSubmitBlock*)dataBuf->pData; + SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)dataBuf->pData; // size is less than the total size, since duplicated rows may be removed yet. assert(pBlocks->numOfRows * dataBuf->rowSize + sizeof(SShellSubmitBlock) == dataBuf->size); @@ -612,26 +640,38 @@ void sortRemoveDuplicates(STableDataBlocks *dataBuf) { dataBuf->ordered = true; pBlocks->numOfRows = i + 1; - dataBuf->size = sizeof(SShellSubmitBlock) + dataBuf->rowSize*pBlocks->numOfRows; + dataBuf->size = sizeof(SShellSubmitBlock) + dataBuf->rowSize * pBlocks->numOfRows; } } static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char **str, SParsedDataColInfo *spd, int32_t *totalNum) { - SSqlCmd * pCmd = &pSql->cmd; - SMeterMeta *pMeterMeta = pCmd->pMeterMeta; + SSqlCmd * pCmd = &pSql->cmd; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + SMeterMeta * pMeterMeta = pMeterMetaInfo->pMeterMeta; STableDataBlocks *dataBuf = tscGetDataBlockFromList(pTableHashList, pCmd->pDataBlocks, pMeterMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SShellSubmitBlock), pMeterMeta->rowSize, pCmd->name); + sizeof(SShellSubmitBlock), pMeterMeta->rowSize, pMeterMetaInfo->name); int32_t maxNumOfRows = tscAllocateMemIfNeed(dataBuf, pMeterMeta->rowSize); + if (0 == maxNumOfRows) { + return TSDB_CODE_CLI_OUT_OF_MEMORY; + } int32_t numOfRows = tsParseValues(str, dataBuf, pMeterMeta, maxNumOfRows, spd, pCmd->payload); if (numOfRows <= 0) { return TSDB_CODE_INVALID_SQL; } + for (uint32_t i = 0; i < dataBuf->numOfParams; ++i) { + SParamInfo* param = dataBuf->params + i; + if (param->idx == -1) { + param->idx = pCmd->numOfParams++; + param->offset -= sizeof(SShellSubmitBlock); + } + } + SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)(dataBuf->pData); tsSetBlockInfo(pBlocks, pMeterMeta, numOfRows); @@ -647,109 +687,121 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char } static int32_t tscParseSqlForCreateTableOnDemand(char **sqlstr, SSqlObj *pSql) { - char * id = NULL; - int32_t idlen = 0; - int32_t code = TSDB_CODE_SUCCESS; + int32_t index = 0; + SSQLToken sToken; + SSQLToken tableToken; + int32_t code = TSDB_CODE_SUCCESS; - SSqlCmd *pCmd = &pSql->cmd; - char * sql = *sqlstr; + SSqlCmd * pCmd = &pSql->cmd; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); - sql = tscGetToken(sql, &id, &idlen); - - /* build the token of specified table */ - SSQLToken tableToken = {.z = id, .n = idlen, .type = TK_ID}; + char *sql = *sqlstr; + // get the token of specified table + index = 0; + tableToken = tStrGetToken(sql, &index, false, 0, NULL); + sql += index; char *cstart = NULL; char *cend = NULL; - /* skip possibly exists column list */ - sql = tscGetToken(sql, &id, &idlen); + // skip possibly exists column list + index = 0; + sToken = tStrGetToken(sql, &index, false, 0, NULL); + sql += index; + int32_t numOfColList = 0; - bool createTable = false; + bool createTable = false; - if (id[0] == '(' && idlen == 1) { - cstart = &id[0]; + if (sToken.type == TK_LP) { + cstart = &sToken.z[0]; + index = 0; while (1) { - sql = tscGetToken(sql, &id, &idlen); - if (id[0] == ')' && idlen == 1) { - cend = &id[0]; + sToken = tStrGetToken(sql, &index, false, 0, NULL); + if (sToken.type == TK_RP) { + cend = &sToken.z[0]; break; } ++numOfColList; } - sql = tscGetToken(sql, &id, &idlen); + sToken = tStrGetToken(sql, &index, false, 0, NULL); + sql += index; } if (numOfColList == 0 && cstart != NULL) { return TSDB_CODE_INVALID_SQL; } - if (strncmp(id, "using", idlen) == 0 && idlen == 5) { - /* create table if not exists */ - sql = tscGetToken(sql, &id, &idlen); + if (sToken.type == TK_USING) { + // create table if not exists + index = 0; + sToken = tStrGetToken(sql, &index, false, 0, NULL); + sql += index; + STagData *pTag = (STagData *)pCmd->payload; memset(pTag, 0, sizeof(STagData)); + setMeterID(pSql, &sToken, 0); - SSQLToken token1 = {idlen, TK_ID, id}; - setMeterID(pSql, &token1); - - strcpy(pTag->name, pSql->cmd.name); - - code = tscGetMeterMeta(pSql, pTag->name); + strncpy(pTag->name, pMeterMetaInfo->name, TSDB_METER_ID_LEN); + code = tscGetMeterMeta(pSql, pTag->name, 0); if (code != TSDB_CODE_SUCCESS) { return code; } - if (!UTIL_METER_IS_METRIC(pCmd)) { - const char* msg = "create table only from super table is allowed"; - sprintf(pCmd->payload, "%s", msg); + if (!UTIL_METER_IS_METRIC(pMeterMetaInfo)) { + strcpy(pCmd->payload, "create table only from super table is allowed"); return TSDB_CODE_INVALID_SQL; } char * tagVal = pTag->data; - SSchema *pTagSchema = tsGetTagSchema(pCmd->pMeterMeta); + SSchema *pTagSchema = tsGetTagSchema(pMeterMetaInfo->pMeterMeta); - sql = tscGetToken(sql, &id, &idlen); - if (!(strncmp(id, "tags", idlen) == 0 && idlen == 4)) { + index = 0; + sToken = tStrGetToken(sql, &index, false, 0, NULL); + sql += index; + if (sToken.type != TK_TAGS) { setErrMsg(pCmd->payload, sql); return TSDB_CODE_INVALID_SQL; } - int32_t numOfTagValues = 0; + int32_t numOfTagValues = 0; + uint32_t ignoreTokenTypes = TK_LP; + uint32_t numOfIgnoreToken = 1; while (1) { - sql = tscGetToken(sql, &id, &idlen); - if (idlen == 0) { + index = 0; + sToken = tStrGetToken(sql, &index, true, numOfIgnoreToken, &ignoreTokenTypes); + sql += index; + if (sToken.n == 0) { break; - } else if (idlen == 1) { - if (id[0] == '(') { - continue; - } - - if (id[0] == ')') { - break; - } - - if (id[0] == '-' || id[0] == '+') { - sql = tscGetToken(sql, &id, &idlen); + } else if (sToken.type == TK_RP) { + break; + } - id -= 1; - idlen += 1; - } + // Remove quotation marks + if (TK_STRING == sToken.type) { + sToken.z++; + sToken.n -= 2; } - code = tsParseOneColumnData(&pTagSchema[numOfTagValues], id, idlen, tagVal, pCmd->payload, &sql, false, - pCmd->pMeterMeta->precision); + code = tsParseOneColumnData(&pTagSchema[numOfTagValues], &sToken, tagVal, pCmd->payload, &sql, false, + pMeterMetaInfo->pMeterMeta->precision); if (code != TSDB_CODE_SUCCESS) { setErrMsg(pCmd->payload, sql); return TSDB_CODE_INVALID_SQL; } + if ((pTagSchema[numOfTagValues].type == TSDB_DATA_TYPE_BINARY || + pTagSchema[numOfTagValues].type == TSDB_DATA_TYPE_NCHAR) && + sToken.n > pTagSchema[numOfTagValues].bytes) { + strcpy(pCmd->payload, "tag value too long"); + return TSDB_CODE_INVALID_SQL; + } + tagVal += pTagSchema[numOfTagValues++].bytes; } - if (numOfTagValues != pCmd->pMeterMeta->numOfTags) { + if (numOfTagValues != pMeterMetaInfo->pMeterMeta->numOfTags) { setErrMsg(pCmd->payload, sql); return TSDB_CODE_INVALID_SQL; } @@ -759,20 +811,20 @@ static int32_t tscParseSqlForCreateTableOnDemand(char **sqlstr, SSqlObj *pSql) { return TSDB_CODE_INVALID_SQL; } - int32_t ret = setMeterID(pSql, &tableToken); + int32_t ret = setMeterID(pSql, &tableToken, 0); if (ret != TSDB_CODE_SUCCESS) { return ret; } createTable = true; - code = tscGetMeterMetaEx(pSql, pSql->cmd.name, true); + code = tscGetMeterMetaEx(pSql, pMeterMetaInfo->name, true); } else { if (cstart != NULL) { sql = cstart; } else { - sql = id; + sql = sToken.z; } - code = tscGetMeterMeta(pSql, pCmd->name); + code = tscGetMeterMeta(pSql, pMeterMetaInfo->name, 0); } int32_t len = cend - cstart + 1; @@ -800,7 +852,6 @@ int validateTableName(char *tblName, int len) { /** * usage: insert into table1 values() () table2 values()() * - * @param pCmd * @param str * @param acct * @param db @@ -817,20 +868,11 @@ int tsParseInsertStatement(SSqlObj *pSql, char *str, char *acct, char *db) { pSql->res.numOfRows = 0; int32_t totalNum = 0; - if (!pSql->pTscObj->writeAuth) { - return TSDB_CODE_NO_RIGHTS; - } - - char *id; - int idlen; - int code = TSDB_CODE_INVALID_SQL; + int code = TSDB_CODE_INVALID_SQL; - str = tscGetToken(str, &id, &idlen); - if (idlen == 0 || (strncmp(id, "into", 4) != 0 || idlen != 4)) { - INVALID_SQL_RET_MSG(pCmd->payload, "keyword INTO is expected"); - } + SMeterMetaInfo *pMeterMetaInfo = tscAddEmptyMeterMetaInfo(pCmd); - if ((code = tscAllocPayloadWithSize(pCmd, TSDB_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { + if ((code = tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { return code; } @@ -840,10 +882,9 @@ int tsParseInsertStatement(SSqlObj *pSql, char *str, char *acct, char *db) { tscTrace("%p create data block list for submit data, %p", pSql, pSql->cmd.pDataBlocks); while (1) { - tscGetToken(str, &id, &idlen); - - if (idlen == 0) { - // parse file, do not release the STableDataBlock + int32_t index = 0; + SSQLToken sToken = tStrGetToken(str, &index, false, 0, NULL); + if (sToken.n == 0) { // parse file, do not release the STableDataBlock if (pCmd->isInsertFromFile == 1) { goto _clean; } @@ -857,14 +898,13 @@ int tsParseInsertStatement(SSqlObj *pSql, char *str, char *acct, char *db) { } // Check if the table name available or not - if (validateTableName(id, idlen) != TSDB_CODE_SUCCESS) { + if (validateTableName(sToken.z, sToken.n) != TSDB_CODE_SUCCESS) { code = TSDB_CODE_INVALID_SQL; sprintf(pCmd->payload, "table name is invalid"); goto _error_clean; } - SSQLToken token = {idlen, TK_ID, id}; - if ((code = setMeterID(pSql, &token)) != TSDB_CODE_SUCCESS) { + if ((code = setMeterID(pSql, &sToken, 0)) != TSDB_CODE_SUCCESS) { goto _error_clean; } @@ -882,24 +922,26 @@ int tsParseInsertStatement(SSqlObj *pSql, char *str, char *acct, char *db) { } } - if (UTIL_METER_IS_METRIC(pCmd)) { + if (UTIL_METER_IS_METRIC(pMeterMetaInfo)) { code = TSDB_CODE_INVALID_SQL; sprintf(pCmd->payload, "insert data into metric is not supported"); goto _error_clean; } - str = tscGetToken(str, &id, &idlen); - if (idlen == 0) { + index = 0; + sToken = tStrGetToken(str, &index, false, 0, NULL); + str += index; + if (sToken.n == 0) { code = TSDB_CODE_INVALID_SQL; sprintf(pCmd->payload, "keyword VALUES or FILE are required"); goto _error_clean; } - if (strncmp(id, "values", 6) == 0 && idlen == 6) { - SParsedDataColInfo spd = {0}; - SSchema * pSchema = tsGetSchema(pCmd->pMeterMeta); + if (sToken.type == TK_VALUES) { + SParsedDataColInfo spd = {.numOfCols = pMeterMetaInfo->pMeterMeta->numOfColumns}; + SSchema * pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); - tscSetAssignedColumnInfo(&spd, pSchema, pCmd->pMeterMeta->numOfColumns); + tscSetAssignedColumnInfo(&spd, pSchema, pMeterMetaInfo->pMeterMeta->numOfColumns); if (pCmd->isInsertFromFile == -1) { pCmd->isInsertFromFile = 0; @@ -920,7 +962,7 @@ int tsParseInsertStatement(SSqlObj *pSql, char *str, char *acct, char *db) { goto _error_clean; } - } else if (strncmp(id, "file", 4) == 0 && idlen == 4) { + } else if (sToken.type == TK_FILE) { if (pCmd->isInsertFromFile == -1) { pCmd->isInsertFromFile = 1; } else { @@ -931,15 +973,17 @@ int tsParseInsertStatement(SSqlObj *pSql, char *str, char *acct, char *db) { } } - str = tscGetTokenDelimiter(str, &id, &idlen, " ;"); - if (idlen == 0) { + index = 0; + sToken = tStrGetToken(str, &index, false, 0, NULL); + str += index; + if (sToken.n == 0) { code = TSDB_CODE_INVALID_SQL; - sprintf(pCmd->payload, "filename is required following keyword FILE"); + sprintf(pCmd->payload, "file path is required following keyword FILE"); goto _error_clean; } char fname[PATH_MAX] = {0}; - strncpy(fname, id, idlen); + strncpy(fname, sToken.z, sToken.n); strdequote(fname); wordexp_t full_path; @@ -951,15 +995,14 @@ int tsParseInsertStatement(SSqlObj *pSql, char *str, char *acct, char *db) { strcpy(fname, full_path.we_wordv[0]); wordfree(&full_path); - STableDataBlocks* pDataBlock = tscCreateDataBlockEx(PATH_MAX, pCmd->pMeterMeta->rowSize, sizeof(SShellSubmitBlock), - pCmd->name); + STableDataBlocks *pDataBlock = tscCreateDataBlockEx(PATH_MAX, pMeterMetaInfo->pMeterMeta->rowSize, + sizeof(SShellSubmitBlock), pMeterMetaInfo->name); tscAppendDataBlock(pCmd->pDataBlocks, pDataBlock); strcpy(pDataBlock->filename, fname); - str = id + idlen; - } else if (idlen == 1 && id[0] == '(') { + } else if (sToken.type == TK_LP) { /* insert into tablename(col1, col2,..., coln) values(v1, v2,... vn); */ - SMeterMeta *pMeterMeta = pCmd->pMeterMeta; + SMeterMeta *pMeterMeta = tscGetMeterMetaInfo(pCmd, 0)->pMeterMeta; SSchema * pSchema = tsGetSchema(pMeterMeta); if (pCmd->isInsertFromFile == -1) { @@ -979,8 +1022,17 @@ int tsParseInsertStatement(SSqlObj *pSql, char *str, char *acct, char *db) { } while (1) { - str = tscGetToken(str, &id, &idlen); - if (idlen == 1 && id[0] == ')') { + index = 0; + sToken = tStrGetToken(str, &index, false, 0, NULL); + str += index; + + if (TK_STRING == sToken.type) { + sToken.n = strdequote(sToken.z); + strtrim(sToken.z); + sToken.n = (uint32_t)strlen(sToken.z); + } + + if (sToken.type == TK_RP) { break; } @@ -988,7 +1040,7 @@ int tsParseInsertStatement(SSqlObj *pSql, char *str, char *acct, char *db) { // todo speedup by using hash list for (int32_t t = 0; t < pMeterMeta->numOfColumns; ++t) { - if (strncmp(id, pSchema[t].name, idlen) == 0 && strlen(pSchema[t].name) == idlen) { + if (strncmp(sToken.z, pSchema[t].name, sToken.n) == 0 && strlen(pSchema[t].name) == sToken.n) { SParsedColElem *pElem = &spd.elems[spd.numOfAssignedCols++]; pElem->offset = offset[t]; pElem->colIndex = t; @@ -1005,7 +1057,7 @@ int tsParseInsertStatement(SSqlObj *pSql, char *str, char *acct, char *db) { } } - if (!findColumnIndex) { // + if (!findColumnIndex) { code = TSDB_CODE_INVALID_SQL; sprintf(pCmd->payload, "invalid column name"); goto _error_clean; @@ -1018,8 +1070,11 @@ int tsParseInsertStatement(SSqlObj *pSql, char *str, char *acct, char *db) { goto _error_clean; } - str = tscGetToken(str, &id, &idlen); - if (strncmp(id, "values", idlen) != 0 || idlen != 6) { + index = 0; + sToken = tStrGetToken(str, &index, false, 0, NULL); + str += index; + + if (sToken.type != TK_VALUES) { code = TSDB_CODE_INVALID_SQL; sprintf(pCmd->payload, "keyword VALUES is expected"); goto _error_clean; @@ -1036,17 +1091,25 @@ int tsParseInsertStatement(SSqlObj *pSql, char *str, char *acct, char *db) { } } + // we need to keep the data blocks if there are parameters in the sql + if (pCmd->numOfParams > 0) { + goto _clean; + } + // submit to more than one vnode if (pCmd->pDataBlocks->nSize > 0) { // merge according to vgid - tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks); + if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { + goto _error_clean; + } STableDataBlocks *pDataBlock = pCmd->pDataBlocks->pData[0]; if ((code = tscCopyDataBlockToPayload(pSql, pDataBlock)) != TSDB_CODE_SUCCESS) { goto _error_clean; } - pCmd->vnodeIdx = 1; // set the next sent data vnode index in data block arraylist + // set the next sent data vnode index in data block arraylist + pCmd->vnodeIdx = 1; } else { pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); } @@ -1062,38 +1125,41 @@ _clean: return code; } -int tsParseImportStatement(SSqlObj *pSql, char *str, char *acct, char *db) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->order.order = TSQL_SO_ASC; - return tsParseInsertStatement(pSql, str, acct, db); -} - int tsParseInsertSql(SSqlObj *pSql, char *sql, char *acct, char *db) { - char *verb; - int verblen; - int code = TSDB_CODE_INVALID_SQL; + if (!pSql->pTscObj->writeAuth) { + return TSDB_CODE_NO_RIGHTS; + } + int32_t index = 0; SSqlCmd *pCmd = &pSql->cmd; - sql = tscGetToken(sql, &verb, &verblen); - if (verblen) { - if (strncmp(verb, "insert", 6) == 0 && verblen == 6) { - code = tsParseInsertStatement(pSql, sql, acct, db); - } else if (strncmp(verb, "import", 6) == 0 && verblen == 6) { - code = tsParseImportStatement(pSql, sql, acct, db); + SSQLToken sToken = tStrGetToken(sql, &index, false, 0, NULL); + if (sToken.type == TK_IMPORT) { + pCmd->order.order = TSQL_SO_ASC; + } else if (sToken.type != TK_INSERT) { + if (sToken.n) { + sToken.z[sToken.n] = 0; + sprintf(pCmd->payload, "invalid keyword:%s", sToken.z); } else { - verb[verblen] = 0; - sprintf(pCmd->payload, "invalid keyword:%s", verb); + strcpy(pCmd->payload, "no any keywords"); } - } else { - sprintf(pCmd->payload, "no any keywords"); + return TSDB_CODE_INVALID_SQL; } - return code; + sToken = tStrGetToken(sql, &index, false, 0, NULL); + if (sToken.type != TK_INTO) { + strcpy(pCmd->payload, "keyword INTO is expected"); + return TSDB_CODE_INVALID_SQL; + } + + return tsParseInsertStatement(pSql, sql + index, acct, db); } int tsParseSql(SSqlObj *pSql, char *acct, char *db, bool multiVnodeInsertion) { int32_t ret = TSDB_CODE_SUCCESS; + + // must before clean the sqlcmd object + tscRemoveAllMeterMetaInfo(&pSql->cmd, false); tscCleanSqlCmd(&pSql->cmd); if (tscIsInsertOrImportData(pSql->sqlstr)) { @@ -1106,17 +1172,18 @@ int tsParseSql(SSqlObj *pSql, char *acct, char *db, bool multiVnodeInsertion) { assert(pSql->fetchFp == NULL); pSql->fetchFp = pSql->fp; - /* replace user defined callback function with multi-insert proxy function */ + // replace user defined callback function with multi-insert proxy function pSql->fp = tscAsyncInsertMultiVnodesProxy; } ret = tsParseInsertSql(pSql, pSql->sqlstr, acct, db); } else { + ret = tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE); + if (TSDB_CODE_SUCCESS != ret) return ret; + SSqlInfo SQLInfo = {0}; tSQLParse(&SQLInfo, pSql->sqlstr); - tscAllocPayloadWithSize(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE); - ret = tscToSQLCmd(pSql, &SQLInfo); SQLInfoDestroy(&SQLInfo); } @@ -1132,16 +1199,18 @@ int tsParseSql(SSqlObj *pSql, char *acct, char *db, bool multiVnodeInsertion) { return ret; } -static int doPackSendDataBlock(SSqlObj* pSql, int32_t numOfRows, STableDataBlocks* pTableDataBlocks) { - int32_t code = TSDB_CODE_SUCCESS; - SSqlCmd* pCmd = &pSql->cmd; +static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlocks *pTableDataBlocks) { + int32_t code = TSDB_CODE_SUCCESS; + SSqlCmd *pCmd = &pSql->cmd; - SMeterMeta* pMeterMeta = pCmd->pMeterMeta; + SMeterMeta *pMeterMeta = tscGetMeterMetaInfo(pCmd, 0)->pMeterMeta; SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)(pTableDataBlocks->pData); tsSetBlockInfo(pBlocks, pMeterMeta, numOfRows); - tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks); + if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { + return code; + } // the pDataBlock is different from the pTableDataBlocks STableDataBlocks *pDataBlock = pCmd->pDataBlocks->pData[0]; @@ -1157,21 +1226,22 @@ static int doPackSendDataBlock(SSqlObj* pSql, int32_t numOfRows, STableDataBlock } static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp) { - int readLen = 0; - char * line = NULL; - size_t n = 0; - int len = 0; - uint32_t maxRows = 0; - SSqlCmd * pCmd = &pSql->cmd; - SMeterMeta *pMeterMeta = pCmd->pMeterMeta; - int numOfRows = 0; - int32_t rowSize = pMeterMeta->rowSize; - int32_t code = 0; - int nrows = 0; + size_t readLen = 0; + char * line = NULL; + size_t n = 0; + int len = 0; + uint32_t maxRows = 0; + SSqlCmd * pCmd = &pSql->cmd; + int numOfRows = 0; + int32_t code = 0; + int nrows = 0; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + SMeterMeta * pMeterMeta = pMeterMetaInfo->pMeterMeta; + int32_t rowSize = pMeterMeta->rowSize; pCmd->pDataBlocks = tscCreateBlockArrayList(); - STableDataBlocks* pTableDataBlock = tscCreateDataBlockEx(TSDB_PAYLOAD_SIZE, pMeterMeta->rowSize, - sizeof(SShellSubmitBlock), pCmd->name); + STableDataBlocks *pTableDataBlock = + tscCreateDataBlockEx(TSDB_PAYLOAD_SIZE, pMeterMeta->rowSize, sizeof(SShellSubmitBlock), pMeterMetaInfo->name); tscAppendDataBlock(pCmd->pDataBlocks, pTableDataBlock); @@ -1179,10 +1249,10 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp) { if (maxRows < 1) return -1; int count = 0; - SParsedDataColInfo spd = {.numOfCols = pCmd->pMeterMeta->numOfColumns}; - SSchema * pSchema = tsGetSchema(pCmd->pMeterMeta); + SParsedDataColInfo spd = {.numOfCols = pMeterMetaInfo->pMeterMeta->numOfColumns}; + SSchema * pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); - tscSetAssignedColumnInfo(&spd, pSchema, pCmd->pMeterMeta->numOfColumns); + tscSetAssignedColumnInfo(&spd, pSchema, pMeterMetaInfo->pMeterMeta->numOfColumns); while ((readLen = getline(&line, &n, fp)) != -1) { // line[--readLen] = '\0'; @@ -1193,12 +1263,16 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp) { strtolower(line, line); if (numOfRows >= maxRows || pTableDataBlock->size + pMeterMeta->rowSize >= pTableDataBlock->nAllocSize) { - maxRows += tscAllocateMemIfNeed(pTableDataBlock, pMeterMeta->rowSize); + uint32_t tSize = tscAllocateMemIfNeed(pTableDataBlock, pMeterMeta->rowSize); + if (0 == tSize) return (-TSDB_CODE_CLI_OUT_OF_MEMORY); + maxRows += tSize; } len = tsParseOneRowData(&lineptr, pTableDataBlock, pSchema, &spd, pCmd->payload, pMeterMeta->precision); - if (len <= 0) return -1; - + if (len <= 0 || pTableDataBlock->numOfParams > 0) { + pSql->res.code = TSDB_CODE_INVALID_SQL; + return -1; + } pTableDataBlock->size += len; count++; @@ -1240,12 +1314,19 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp) { */ void tscProcessMultiVnodesInsert(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; + + // not insert/import, return directly if (pCmd->command != TSDB_SQL_INSERT) { return; } + // SSqlCmd may have been released + if (pCmd->pDataBlocks == NULL) { + return; + } + STableDataBlocks *pDataBlock = NULL; - int32_t affected_rows = 0; + SMeterMetaInfo * pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); int32_t code = TSDB_CODE_SUCCESS; /* the first block has been sent to server in processSQL function */ @@ -1273,13 +1354,14 @@ void tscProcessMultiVnodesInsert(SSqlObj *pSql) { pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); } -/* multi-vnodes insertion in sync query model */ +// multi-vnodes insertion in sync query model void tscProcessMultiVnodesInsertForFile(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; if (pCmd->command != TSDB_SQL_INSERT) { return; } + SMeterMetaInfo * pInfo = tscGetMeterMetaInfo(pCmd, 0); STableDataBlocks *pDataBlock = NULL; int32_t affected_rows = 0; @@ -1287,30 +1369,32 @@ void tscProcessMultiVnodesInsertForFile(SSqlObj *pSql) { SDataBlockList *pDataBlockList = pCmd->pDataBlocks; pCmd->pDataBlocks = NULL; - char path[PATH_MAX] = {0}; + char path[PATH_MAX] = {0}; for (int32_t i = 0; i < pDataBlockList->nSize; ++i) { pDataBlock = pDataBlockList->pData[i]; if (pDataBlock == NULL) { continue; } - - tscAllocPayloadWithSize(pCmd, TSDB_PAYLOAD_SIZE); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE)) { + tscError("%p failed to malloc when insert file", pSql); + continue; + } pCmd->count = 1; strncpy(path, pDataBlock->filename, PATH_MAX); FILE *fp = fopen(path, "r"); if (fp == NULL) { - tscError("%p failed to open file %s to load data from file, reason:%s", pSql, path, - strerror(errno)); + tscError("%p failed to open file %s to load data from file, reason:%s", pSql, path, strerror(errno)); continue; } - strcpy(pCmd->name, pDataBlock->meterId); + strncpy(pInfo->name, pDataBlock->meterId, TSDB_METER_ID_LEN); memset(pDataBlock->pData, 0, pDataBlock->nAllocSize); - int32_t ret = tscGetMeterMeta(pSql, pCmd->name); + int32_t ret = tscGetMeterMeta(pSql, pInfo->name, 0); if (ret != TSDB_CODE_SUCCESS) { tscError("%p get meter meta failed, abort", pSql); continue; @@ -1321,7 +1405,7 @@ void tscProcessMultiVnodesInsertForFile(SSqlObj *pSql) { if (nrows < 0) { fclose(fp); - tscTrace("%p no records in file %s", pSql, path); + tscTrace("%p no records(%d) in file %s", pSql, nrows, path); continue; } diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c new file mode 100644 index 0000000000..e956d6159e --- /dev/null +++ b/src/client/src/tscPrepare.c @@ -0,0 +1,599 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +#include "taos.h" +#include "tsclient.h" +#include "tsql.h" +#include "tscUtil.h" +#include "ttimer.h" +#include "taosmsg.h" +#include "tstrbuild.h" + + +int tsParseInsertSql(SSqlObj *pSql, char *sql, char *acct, char *db); +int taos_query_imp(STscObj* pObj, SSqlObj* pSql); + +//////////////////////////////////////////////////////////////////////////////// +// functions for normal statement preparation + +typedef struct SNormalStmtPart { + bool isParam; + uint32_t len; + char* str; +} SNormalStmtPart; + +typedef struct SNormalStmt { + uint16_t sizeParts; + uint16_t numParts; + uint16_t numParams; + char* sql; + SNormalStmtPart* parts; + tVariant* params; +} SNormalStmt; + +//typedef struct SInsertStmt { +// +//} SInsertStmt; + +typedef struct STscStmt { + bool isInsert; + STscObj* taos; + SSqlObj* pSql; + SNormalStmt normal; +} STscStmt; + + +static int normalStmtAddPart(SNormalStmt* stmt, bool isParam, char* str, uint32_t len) { + uint16_t size = stmt->numParts + 1; + if (size > stmt->sizeParts) { + size *= 2; + void* tmp = realloc(stmt->parts, sizeof(SNormalStmtPart) * size); + if (tmp == NULL) { + return TSDB_CODE_CLI_OUT_OF_MEMORY; + } + stmt->sizeParts = size; + stmt->parts = (SNormalStmtPart*)tmp; + } + + stmt->parts[stmt->numParts].isParam = isParam; + stmt->parts[stmt->numParts].str = str; + stmt->parts[stmt->numParts].len = len; + + ++stmt->numParts; + if (isParam) { + ++stmt->numParams; + } + + return TSDB_CODE_SUCCESS; +} + +static int normalStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { + SNormalStmt* normal = &stmt->normal; + + for (uint16_t i = 0; i < normal->numParams; ++i) { + TAOS_BIND* tb = bind + i; + tVariant* var = normal->params + i; + tVariantDestroy(var); + + var->nLen = 0; + if (tb->is_null != NULL && *(tb->is_null)) { + var->nType = TSDB_DATA_TYPE_NULL; + var->i64Key = 0; + continue; + } + + var->nType = tb->buffer_type; + switch (tb->buffer_type) { + case TSDB_DATA_TYPE_NULL: + var->i64Key = 0; + break; + + case TSDB_DATA_TYPE_BOOL: + var->i64Key = (*(int8_t*)tb->buffer) ? 1 : 0; + break; + + case TSDB_DATA_TYPE_TINYINT: + var->i64Key = *(int8_t*)tb->buffer; + break; + + case TSDB_DATA_TYPE_SMALLINT: + var->i64Key = *(int16_t*)tb->buffer; + break; + + case TSDB_DATA_TYPE_INT: + var->i64Key = *(int32_t*)tb->buffer; + break; + + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + var->i64Key = *(int64_t*)tb->buffer; + break; + + case TSDB_DATA_TYPE_FLOAT: + var->dKey = *(float*)tb->buffer; + break; + + case TSDB_DATA_TYPE_DOUBLE: + var->dKey = *(double*)tb->buffer; + break; + + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + var->pz = (char*)malloc((*tb->length) + 1); + if (var->pz == NULL) { + return TSDB_CODE_CLI_OUT_OF_MEMORY; + } + memcpy(var->pz, tb->buffer, (*tb->length)); + var->pz[*tb->length] = 0; + var->nLen = (int32_t)(*tb->length); + break; + + default: + tscTrace("param %d: type mismatch or invalid", i); + return TSDB_CODE_INVALID_VALUE; + } + } + + return TSDB_CODE_SUCCESS; +} + + +static int normalStmtPrepare(STscStmt* stmt) { + SNormalStmt* normal = &stmt->normal; + char* sql = stmt->pSql->sqlstr; + uint32_t i = 0, start = 0; + + while (sql[i] != 0) { + SSQLToken token = {0}; + token.n = tSQLGetToken(sql + i, &token.type); + + if (token.type == TK_QUESTION) { + sql[i] = 0; + if (i > start) { + int code = normalStmtAddPart(normal, false, sql + start, i - start); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + int code = normalStmtAddPart(normal, true, NULL, 0); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + start = i + token.n; + } + + i += token.n; + } + + if (i > start) { + int code = normalStmtAddPart(normal, false, sql + start, i - start); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + if (normal->numParams > 0) { + normal->params = calloc(normal->numParams, sizeof(tVariant)); + if (normal->params == NULL) { + return TSDB_CODE_CLI_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + +static char* normalStmtBuildSql(STscStmt* stmt) { + SNormalStmt* normal = &stmt->normal; + SStringBuilder sb = {0}; + + if (taosStringBuilderSetJmp(&sb) != 0) { + taosStringBuilderDestroy(&sb); + return NULL; + } + + taosStringBuilderEnsureCapacity(&sb, 4096); + uint32_t idxParam = 0; + + for(uint16_t i = 0; i < normal->numParts; i++) { + SNormalStmtPart* part = normal->parts + i; + if (!part->isParam) { + taosStringBuilderAppendStringLen(&sb, part->str, part->len); + continue; + } + + tVariant* var = normal->params + idxParam++; + switch (var->nType) + { + case TSDB_DATA_TYPE_NULL: + taosStringBuilderAppendNull(&sb); + break; + + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + taosStringBuilderAppendInteger(&sb, var->i64Key); + break; + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + taosStringBuilderAppendDouble(&sb, var->dKey); + break; + + case TSDB_DATA_TYPE_BINARY: + taosStringBuilderAppendChar(&sb, '\''); + for (char* p = var->pz; *p != 0; ++p) { + if (*p == '\'' || *p == '"') { + taosStringBuilderAppendChar(&sb, '\\'); + } + taosStringBuilderAppendChar(&sb, *p); + } + taosStringBuilderAppendChar(&sb, '\''); + break; + + case TSDB_DATA_TYPE_NCHAR: + taosStringBuilderAppendChar(&sb, '\''); + taosStringBuilderAppend(&sb, var->wpz, var->nLen); + taosStringBuilderAppendChar(&sb, '\''); + break; + + default: + assert(false); + break; + } + } + + return taosStringBuilderGetResult(&sb, NULL); +} + +//////////////////////////////////////////////////////////////////////////////// +// functions for insertion statement preparation + +static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { + if (bind->is_null != NULL && *(bind->is_null)) { + setNull(data, param->type, param->bytes); + return TSDB_CODE_SUCCESS; + } + + if (bind->buffer_type != param->type) { + return TSDB_CODE_INVALID_VALUE; + } + + short size = 0; + switch(param->type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + size = 1; + break; + + case TSDB_DATA_TYPE_SMALLINT: + size = 2; + break; + + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_FLOAT: + size = 4; + break; + + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + size = 8; + break; + + case TSDB_DATA_TYPE_BINARY: + if ((*bind->length) > param->bytes) { + return TSDB_CODE_INVALID_VALUE; + } + size = (short)*bind->length; + break; + + case TSDB_DATA_TYPE_NCHAR: + if (!taosMbsToUcs4(bind->buffer, *bind->length, data + param->offset, param->bytes)) { + return TSDB_CODE_INVALID_VALUE; + } + return TSDB_CODE_SUCCESS; + + default: + assert(false); + return TSDB_CODE_INVALID_VALUE; + } + + memcpy(data + param->offset, bind->buffer, size); + return TSDB_CODE_SUCCESS; +} + +static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { + SSqlCmd* pCmd = &stmt->pSql->cmd; + + int32_t alloced = 1, binded = 0; + if (pCmd->batchSize > 0) { + alloced = (pCmd->batchSize + 1) / 2; + binded = pCmd->batchSize / 2; + } + + for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) { + STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i]; + uint32_t totalDataSize = pBlock->size - sizeof(SShellSubmitBlock); + uint32_t dataSize = totalDataSize / alloced; + assert(dataSize * alloced == totalDataSize); + + if (alloced == binded) { + totalDataSize += dataSize + sizeof(SShellSubmitBlock); + if (totalDataSize > pBlock->nAllocSize) { + const double factor = 1.5; + void* tmp = realloc(pBlock->pData, (uint32_t)(totalDataSize * factor)); + if (tmp == NULL) { + return TSDB_CODE_CLI_OUT_OF_MEMORY; + } + pBlock->pData = (char*)tmp; + pBlock->nAllocSize = (uint32_t)(totalDataSize * factor); + } + } + + char* data = pBlock->pData + sizeof(SShellSubmitBlock) + dataSize * binded; + for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { + SParamInfo* param = pBlock->params + j; + int code = doBindParam(data, param, bind + param->idx); + if (code != TSDB_CODE_SUCCESS) { + tscTrace("param %d: type mismatch or invalid", param->idx); + return code; + } + } + } + + // actual work of all data blocks is done, update block size and numOfRows. + // note we don't do this block by block during the binding process, because + // we cannot recover if something goes wrong. + pCmd->batchSize = binded * 2 + 1; + + if (binded < alloced) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) { + STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i]; + + uint32_t totalDataSize = pBlock->size - sizeof(SShellSubmitBlock); + pBlock->size += totalDataSize / alloced; + + SShellSubmitBlock* pSubmit = (SShellSubmitBlock*)pBlock->pData; + pSubmit->numOfRows += pSubmit->numOfRows / alloced; + } + + return TSDB_CODE_SUCCESS; +} + +static int insertStmtAddBatch(STscStmt* stmt) { + SSqlCmd* pCmd = &stmt->pSql->cmd; + if ((pCmd->batchSize % 2) == 1) { + ++pCmd->batchSize; + } + return TSDB_CODE_SUCCESS; +} + +static int insertStmtPrepare(STscStmt* stmt) { + STscObj* taos = stmt->taos; + SSqlObj *pSql = stmt->pSql; + pSql->cmd.numOfParams = 0; + pSql->cmd.batchSize = 0; + + return tsParseInsertSql(pSql, pSql->sqlstr, taos->acctId, taos->db); +} + +static int insertStmtReset(STscStmt* pStmt) { + SSqlCmd* pCmd = &pStmt->pSql->cmd; + if (pCmd->batchSize > 2) { + int32_t alloced = (pCmd->batchSize + 1) / 2; + for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) { + STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i]; + + uint32_t totalDataSize = pBlock->size - sizeof(SShellSubmitBlock); + pBlock->size = sizeof(SShellSubmitBlock) + totalDataSize / alloced; + + SShellSubmitBlock* pSubmit = (SShellSubmitBlock*)pBlock->pData; + pSubmit->numOfRows = pSubmit->numOfRows / alloced; + } + } + pCmd->batchSize = 0; + pCmd->vnodeIdx = 0; + return TSDB_CODE_SUCCESS; +} + +static int insertStmtExecute(STscStmt* stmt) { + SSqlCmd* pCmd = &stmt->pSql->cmd; + if (pCmd->batchSize == 0) { + return TSDB_CODE_INVALID_VALUE; + } + if ((pCmd->batchSize % 2) == 1) { + ++pCmd->batchSize; + } + + if (pCmd->pDataBlocks->nSize > 0) { + // merge according to vgid + int code = tscMergeTableDataBlocks(stmt->pSql, pCmd->pDataBlocks); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + STableDataBlocks *pDataBlock = pCmd->pDataBlocks->pData[0]; + code = tscCopyDataBlockToPayload(stmt->pSql, pDataBlock); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // set the next sent data vnode index in data block arraylist + pCmd->vnodeIdx = 1; + } else { + pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); + } + + SSqlObj *pSql = stmt->pSql; + SSqlRes *pRes = &pSql->res; + pRes->numOfRows = 0; + pRes->numOfTotal = 0; + pRes->qhandle = 0; + pSql->thandle = NULL; + + tscDoQuery(pSql); + + // tscTrace("%p SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj); + if (pRes->code != TSDB_CODE_SUCCESS) { + tscFreeSqlObjPartial(pSql); + } + + return pRes->code; +} + +//////////////////////////////////////////////////////////////////////////////// +// interface functions + +TAOS_STMT* taos_stmt_init(TAOS* taos) { + STscObj* pObj = (STscObj*)taos; + if (pObj == NULL || pObj->signature != pObj) { + globalCode = TSDB_CODE_DISCONNECTED; + tscError("connection disconnected"); + return NULL; + } + + STscStmt* pStmt = calloc(1, sizeof(STscStmt)); + if (pStmt == NULL) { + globalCode = TSDB_CODE_CLI_OUT_OF_MEMORY; + tscError("failed to allocate memory for statement"); + return NULL; + } + + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + if (pSql == NULL) { + free(pStmt); + globalCode = TSDB_CODE_CLI_OUT_OF_MEMORY; + tscError("failed to allocate memory for statement"); + return NULL; + } + + tsem_init(&pSql->rspSem, 0, 0); + tsem_init(&pSql->emptyRspSem, 0, 1); + pSql->signature = pSql; + pSql->pTscObj = pObj; + + pStmt->pSql = pSql; + return pStmt; +} + +int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { + STscStmt* pStmt = (STscStmt*)stmt; + if (length == 0) { + length = strlen(sql); + } + char* sqlstr = (char*)malloc(length + 1); + if (sqlstr == NULL) { + tscError("failed to malloc sql string buffer"); + return TSDB_CODE_CLI_OUT_OF_MEMORY; + } + memcpy(sqlstr, sql, length); + sqlstr[length] = 0; + strtolower(sqlstr, sqlstr); + + pStmt->pSql->sqlstr = sqlstr; + if (tscIsInsertOrImportData(sqlstr)) { + pStmt->isInsert = true; + return insertStmtPrepare(pStmt); + } + + pStmt->isInsert = false; + return normalStmtPrepare(pStmt); +} + +int taos_stmt_close(TAOS_STMT* stmt) { + STscStmt* pStmt = (STscStmt*)stmt; + if (!pStmt->isInsert) { + SNormalStmt* normal = &pStmt->normal; + if (normal->params != NULL) { + for (uint16_t i = 0; i < normal->numParams; i++) { + tVariantDestroy(normal->params + i); + } + free(normal->params); + } + free(normal->parts); + free(normal->sql); + } + + tscFreeSqlObj(pStmt->pSql); + free(pStmt); + return TSDB_CODE_SUCCESS; +} + +int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { + STscStmt* pStmt = (STscStmt*)stmt; + if (pStmt->isInsert) { + return insertStmtBindParam(pStmt, bind); + } + return normalStmtBindParam(pStmt, bind); +} + +int taos_stmt_add_batch(TAOS_STMT* stmt) { + STscStmt* pStmt = (STscStmt*)stmt; + if (pStmt->isInsert) { + return insertStmtAddBatch(pStmt); + } + return TSDB_CODE_OPS_NOT_SUPPORT; +} + +int taos_stmt_reset(TAOS_STMT* stmt) { + STscStmt* pStmt = (STscStmt*)stmt; + if (pStmt->isInsert) { + return insertStmtReset(pStmt); + } + return TSDB_CODE_SUCCESS; +} + +int taos_stmt_execute(TAOS_STMT* stmt) { + int ret = 0; + STscStmt* pStmt = (STscStmt*)stmt; + if (pStmt->isInsert) { + ret = insertStmtExecute(pStmt); + } else { + char* sql = normalStmtBuildSql(pStmt); + if (sql == NULL) { + ret = TSDB_CODE_CLI_OUT_OF_MEMORY; + } else { + tfree(pStmt->pSql->sqlstr); + pStmt->pSql->sqlstr = sql; + ret = taos_query_imp(pStmt->taos, pStmt->pSql); + } + } + return ret; +} + +TAOS_RES *taos_stmt_use_result(TAOS_STMT* stmt) { + if (stmt == NULL) { + tscError("statement is invalid."); + return NULL; + } + + STscStmt* pStmt = (STscStmt*)stmt; + if (pStmt->pSql == NULL) { + tscError("result has been used already."); + return NULL; + } + + TAOS_RES* result = pStmt->pSql; + pStmt->pSql = NULL; + return result; +} diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f811d62610..ae9704effa 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -16,17 +16,9 @@ #define _XOPEN_SOURCE #define _DEFAULT_SOURCE -#include -#include -#ifdef LINUX -#include -#endif -#include -#include -#include - #include "os.h" #include "taos.h" +#include "taosmsg.h" #include "tstoken.h" #include "ttime.h" @@ -36,19 +28,27 @@ #include "tsql.h" #pragma GCC diagnostic ignored "-Wunused-variable" -typedef struct SColumnIdList { - SSchema* pSchema; - int32_t numOfCols; - int32_t numOfRecordedCols; - int32_t ids[TSDB_MAX_COLUMNS]; -} SColumnIdList; +#define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" + +// -1 is tbname column index, so here use the -2 as the initial value +#define COLUMN_INDEX_INITIAL_VAL (-2) +#define COLUMN_INDEX_INITIALIZER \ + { COLUMN_INDEX_INITIAL_VAL, COLUMN_INDEX_INITIAL_VAL } +#define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_TBNAME_COLUMN_INDEX)) +#define TBNAME_LIST_SEP "," typedef struct SColumnList { - int32_t numOfCols; - int32_t ids[TSDB_MAX_COLUMNS]; + int32_t num; + SColumnIndex ids[TSDB_MAX_COLUMNS]; } SColumnList; -static void setProjExprForMetricQuery(SSqlCmd* pCmd, int32_t fieldIDInResult, int32_t colIdx); +typedef struct SColumnIdListRes { + SSchema* pSchema; + int32_t numOfCols; + SColumnList list; +} SColumnIdListRes; + +static SSqlExpr* doAddProjectCol(SSqlCmd* pCmd, int32_t fieldIDInResult, int32_t colIdx, int32_t tableIndex); static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); @@ -64,8 +64,6 @@ static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSq static int32_t setObjFullName(char* fullName, char* account, SSQLToken* pDB, SSQLToken* tableName, int32_t* len); -static int32_t getColumnIndexByName(SSQLToken* pToken, SSchema* pSchema, int32_t numOfCols); - static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nLen); static void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName); @@ -73,55 +71,63 @@ static int32_t addExprAndResultField(SSqlCmd* pCmd, int32_t colIdx, tSQLExprItem static int32_t insertResultField(SSqlCmd* pCmd, int32_t fieldIDInResult, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName); -static int32_t changeFunctionID(int32_t optr, int16_t* pExpr); +static int32_t changeFunctionID(int32_t optr, int16_t* functionId); static void setErrMsg(SSqlCmd* pCmd, const char* pzErrMsg); -static int32_t buildSelectionClause(SSqlCmd* pCmd, tSQLExprList* pSelection, bool isMetric); +static int32_t parseSelectClause(SSqlCmd* pCmd, tSQLExprList* pSelection, bool isMetric); static bool validateIpAddress(char* ip); -static bool onlyQueryMetricTags(SSqlCmd* pCmd); static bool hasUnsupportFunctionsForMetricQuery(SSqlCmd* pCmd); static bool functionCompatibleCheck(SSqlCmd* pCmd); -static void setColumnOffsetValueInResultset(SSqlCmd* pCmd); -static int32_t setGroupByClause(SSqlCmd* pCmd, tVariantList* pList); +static void setColumnOffsetValueInResultset(SSqlCmd* pCmd); +static int32_t parseGroupbyClause(SSqlCmd* pCmd, tVariantList* pList); -static int32_t setIntervalClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql); +static int32_t parseIntervalClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql); static int32_t setSlidingClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql); -static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SSchema* pSchema, tSQLExprItem* pItem, bool isMet); +static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, tSQLExprItem* pItem); -static int32_t buildQueryCond(SSqlObj* pSql, tSQLExpr* pExpr); -static int32_t setFillPolicy(SSqlCmd* pCmd, SQuerySQL* pQuerySQL); -static int32_t setOrderByClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql, SSchema* pSchema, int32_t numOfCols); +static int32_t parseWhereClause(SSqlObj* pSql, tSQLExpr** pExpr); +static int32_t parseFillClause(SSqlCmd* pCmd, SQuerySQL* pQuerySQL); +static int32_t parseOrderbyClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql, SSchema* pSchema, int32_t numOfCols); static int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd); -static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField); +static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField); static int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); static int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd); -static int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString, SColumnIdList* colIdList); +static int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString); static int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd); -static int32_t validateArithmeticSQLExpr(tSQLExpr* pExpr, SSchema* pSchema, int32_t numOfCols); +static int32_t validateArithmeticSQLExpr(tSQLExpr* pExpr, SSchema* pSchema, int32_t numOfCols, SColumnIdListRes* pList); static int32_t validateDNodeConfig(tDCLSQL* pOptions); +static int32_t validateLocalConfig(tDCLSQL* pOptions); static int32_t validateColumnName(char* name); static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); -static bool hasTimestampForPointInterpQuery(SSqlCmd* pCmd); -static void updateTagColumnIndex(SSqlCmd* pCmd); + +static bool hasTimestampForPointInterpQuery(SSqlCmd* pCmd); +static void updateTagColumnIndex(SSqlCmd* pCmd, int32_t tableIndex); static int32_t parseLimitClause(SSqlObj* pSql, SQuerySQL* pQuerySql); -static void addRequiredTagColumn(SSqlCmd* pCmd, int32_t tagColIndex); + static int32_t parseCreateDBOptions(SCreateDBInfo* pCreateDbSql, SSqlCmd* pCmd); +static int32_t getColumnIndexByNameEx(SSQLToken* pToken, SSqlCmd* pCmd, SColumnIndex* pIndex); +static int32_t getTableIndexByName(SSQLToken* pToken, SSqlCmd* pCmd, SColumnIndex* pIndex); +static int32_t optrToString(tSQLExpr* pExpr, char** exprString); + +static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex); +static int32_t getMeterIndex(SSQLToken* pTableToken, SSqlCmd* pCmd, SColumnIndex* pIndex); +static int32_t doFunctionsCompatibleCheck(SSqlObj* pSql); static int32_t tscQueryOnlyMetricTags(SSqlCmd* pCmd, bool* queryOnMetricTags) { - assert(pCmd->metricQuery == 1); + assert(QUERY_IS_STABLE_QUERY(pCmd->type)); // here colIdx == -1 means the special column tbname that is the name of each table *queryOnMetricTags = true; for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); - if (pExpr->sqlFuncId != TSDB_FUNC_TAGPRJ && - !(pExpr->sqlFuncId == TSDB_FUNC_COUNT && pExpr->colInfo.colIdx == -1)) { // 23 == "tagprj" function + if (pExpr->functionId != TSDB_FUNC_TAGPRJ && + !(pExpr->functionId == TSDB_FUNC_COUNT && pExpr->colInfo.colIdx == TSDB_TBNAME_COLUMN_INDEX)) { *queryOnMetricTags = false; break; } @@ -130,6 +136,32 @@ static int32_t tscQueryOnlyMetricTags(SSqlCmd* pCmd, bool* queryOnMetricTags) { return TSDB_CODE_SUCCESS; } +static int setColumnFilterInfoForTimestamp(SSqlCmd* pCmd, tVariant* pVar) { + int64_t time = 0; + const char* msg = "invalid timestamp"; + + strdequote(pVar->pz); + char* seg = strnchr(pVar->pz, '-', pVar->nLen, false); + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + + if (seg != NULL) { + if (taosParseTime(pVar->pz, &time, pVar->nLen, pMeterMetaInfo->pMeterMeta->precision) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg); + return TSDB_CODE_INVALID_SQL; + } + } else { + if (tVariantDump(pVar, (char*)&time, TSDB_DATA_TYPE_BIGINT)) { + setErrMsg(pCmd, msg); + return TSDB_CODE_INVALID_SQL; + } + } + + tVariantDestroy(pVar); + tVariantCreateFromBinary(pVar, (char*)&time, 0, TSDB_DATA_TYPE_BIGINT); + + return TSDB_CODE_SUCCESS; +} + // todo handle memory leak in error handle function int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pInfo == NULL || pSql == NULL || pSql->signature != pSql) { @@ -143,11 +175,14 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } + SMeterMetaInfo* pMeterMetaInfo = tscAddEmptyMeterMetaInfo(pCmd); + // transfer pInfo into select operation switch (pInfo->sqlType) { case DROP_TABLE: case DROP_USER: case DROP_ACCOUNT: + case DROP_DNODE: case DROP_DATABASE: { const char* msg = "param name too long"; const char* msg1 = "invalid ip address"; @@ -165,7 +200,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { pCmd->command = TSDB_SQL_DROP_DB; pCmd->existsCheck = (pInfo->pDCLInfo->a[1].n == 1); - int32_t code = setObjFullName(pCmd->name, getAccountId(pSql), pzName, NULL, NULL); + int32_t code = setObjFullName(pMeterMetaInfo->name, getAccountId(pSql), pzName, NULL, NULL); if (code != TSDB_CODE_SUCCESS) { setErrMsg(pCmd, msg2); } @@ -177,7 +212,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { pCmd->existsCheck = (pInfo->pDCLInfo->a[1].n == 1); pCmd->command = TSDB_SQL_DROP_TABLE; - int32_t ret = setMeterID(pSql, pzName); + int32_t ret = setMeterID(pSql, pzName, 0); if (ret != TSDB_CODE_SUCCESS) { setErrMsg(pCmd, msg); } @@ -190,9 +225,26 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pInfo->sqlType == DROP_USER) { pCmd->command = TSDB_SQL_DROP_USER; + } else if (pInfo->sqlType == DROP_ACCOUNT) { + pCmd->command = TSDB_SQL_DROP_ACCT; + } else if (pInfo->sqlType == DROP_DNODE) { + pCmd->command = TSDB_SQL_DROP_DNODE; + const int32_t MAX_IP_ADDRESS_LEGNTH = 16; + + if (pzName->n > MAX_IP_ADDRESS_LEGNTH) { + setErrMsg(pCmd, msg1); + return TSDB_CODE_INVALID_SQL; + } + + char str[128] = {0}; + strncpy(str, pzName->z, pzName->n); + if (!validateIpAddress(str)) { + setErrMsg(pCmd, msg1); + return TSDB_CODE_INVALID_SQL; + } } - strncpy(pCmd->name, pzName->z, pzName->n); + strncpy(pMeterMetaInfo->name, pzName->z, pzName->n); return TSDB_CODE_SUCCESS; } } @@ -214,7 +266,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - int32_t ret = setObjFullName(pCmd->name, getAccountId(pSql), pToken, NULL, NULL); + int32_t ret = setObjFullName(pMeterMetaInfo->name, getAccountId(pSql), pToken, NULL, NULL); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -230,13 +282,17 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case SHOW_DATABASES: case SHOW_TABLES: case SHOW_STABLES: + case SHOW_MNODES: case SHOW_DNODES: + case SHOW_ACCOUNTS: case SHOW_USERS: case SHOW_VGROUPS: + case SHOW_MODULES: case SHOW_CONNECTIONS: case SHOW_QUERIES: case SHOW_STREAMS: case SHOW_SCORES: + case SHOW_GRANTS: case SHOW_CONFIGS: { return setShowInfo(pSql, pInfo); } @@ -259,7 +315,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - int32_t ret = setObjFullName(pCmd->name, getAccountId(pSql), &(pCreateDB->dbname), NULL, NULL); + int32_t ret = setObjFullName(pMeterMetaInfo->name, getAccountId(pSql), &(pCreateDB->dbname), NULL, NULL); if (ret != TSDB_CODE_SUCCESS) { setErrMsg(pCmd, msg2); return ret; @@ -272,6 +328,29 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { break; } + case CREATE_DNODE: { + // todo parse hostname + pCmd->command = TSDB_SQL_CREATE_DNODE; + const char* msg = "invalid ip address"; + + char ipAddr[64] = {0}; + const int32_t MAX_IP_ADDRESS_LENGTH = 16; + if (pInfo->pDCLInfo->nTokens > 1 || pInfo->pDCLInfo->a[0].n > MAX_IP_ADDRESS_LENGTH) { + setErrMsg(pCmd, msg); + return TSDB_CODE_INVALID_SQL; + } + + memcpy(ipAddr, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n); + if (validateIpAddress(ipAddr) == false) { + setErrMsg(pCmd, msg); + return TSDB_CODE_INVALID_SQL; + } + + strncpy(pMeterMetaInfo->name, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n); + break; + } + + case CREATE_ACCOUNT: case CREATE_USER: { pCmd->command = (pInfo->sqlType == CREATE_USER) ? TSDB_SQL_CREATE_USER : TSDB_SQL_CREATE_ACCT; assert(pInfo->pDCLInfo->nTokens >= 2); @@ -306,8 +385,41 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - strncpy(pCmd->name, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n); // name - strncpy(pCmd->payload, pInfo->pDCLInfo->a[1].z, pInfo->pDCLInfo->a[1].n); // passwd + strncpy(pMeterMetaInfo->name, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n); // name + strncpy(pCmd->payload, pInfo->pDCLInfo->a[1].z, pInfo->pDCLInfo->a[1].n); // passwd + + if (pInfo->sqlType == CREATE_ACCOUNT) { + SCreateAcctSQL* pAcctOpt = &pInfo->pDCLInfo->acctOpt; + + pCmd->defaultVal[0] = pAcctOpt->users; + pCmd->defaultVal[1] = pAcctOpt->dbs; + pCmd->defaultVal[2] = pAcctOpt->tseries; + pCmd->defaultVal[3] = pAcctOpt->streams; + pCmd->defaultVal[4] = pAcctOpt->pps; + pCmd->defaultVal[5] = pAcctOpt->storage; + pCmd->defaultVal[6] = pAcctOpt->qtime; + pCmd->defaultVal[7] = pAcctOpt->conns; + + if (pAcctOpt->stat.n == 0) { + pCmd->defaultVal[8] = -1; + } else { + strdequote(pAcctOpt->stat.z); + pAcctOpt->stat.n = strlen(pAcctOpt->stat.z); + + if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { + pCmd->defaultVal[8] = TSDB_VN_READ_ACCCESS; + } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { + pCmd->defaultVal[8] = TSDB_VN_WRITE_ACCCESS; + } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { + pCmd->defaultVal[8] = TSDB_VN_ALL_ACCCESS; + } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { + pCmd->defaultVal[8] = 0; + } else { + setErrMsg(pCmd, msg4); + return TSDB_CODE_INVALID_SQL; + } + } + } break; } case ALTER_ACCT: { @@ -354,7 +466,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - strncpy(pCmd->name, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n); // name + strncpy(pMeterMetaInfo->name, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n); // name SCreateAcctSQL* pAcctOpt = &pInfo->pDCLInfo->acctOpt; pCmd->defaultVal[0] = pAcctOpt->users; @@ -404,12 +516,12 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - if (setMeterID(pSql, pToken) != TSDB_CODE_SUCCESS) { + if (setMeterID(pSql, pToken, 0) != TSDB_CODE_SUCCESS) { setErrMsg(pCmd, msg); return TSDB_CODE_INVALID_SQL; } - int32_t ret = tscGetMeterMeta(pSql, pSql->cmd.name); + int32_t ret = tscGetMeterMeta(pSql, pMeterMetaInfo->name, 0); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -419,7 +531,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case ALTER_DNODE: case ALTER_USER_PASSWD: case ALTER_USER_PRIVILEGES: { - pCmd->command = (pInfo->sqlType == ALTER_DNODE) ? TSDB_SQL_CFG_PNODE : TSDB_SQL_ALTER_USER; + pCmd->command = (pInfo->sqlType == ALTER_DNODE) ? TSDB_SQL_CFG_DNODE : TSDB_SQL_ALTER_USER; tDCLSQL* pDCL = pInfo->pDCLInfo; @@ -444,7 +556,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - if (pCmd->command == TSDB_SQL_CFG_PNODE) { + if (pCmd->command == TSDB_SQL_CFG_DNODE) { char ip[128] = {0}; strncpy(ip, pDCL->a[0].z, pDCL->a[0].n); @@ -454,7 +566,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - strcpy(pCmd->name, ip); + strcpy(pMeterMetaInfo->name, ip); /* validate the parameter names and options */ if (validateDNodeConfig(pDCL) != TSDB_CODE_SUCCESS) { @@ -472,7 +584,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg = "invalid user rights"; const char* msg1 = "password can not be empty or larger than 24 characters"; - strncpy(pCmd->name, pDCL->a[0].z, pDCL->a[0].n); + strncpy(pMeterMetaInfo->name, pDCL->a[0].z, pDCL->a[0].n); if (pInfo->sqlType == ALTER_USER_PASSWD) { /* update the password for user */ @@ -509,13 +621,27 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case ALTER_LOCAL: { pCmd->command = TSDB_SQL_CFG_LOCAL; - const char* msg = "parameter too long"; + /* if (pInfo->pDCLInfo->a[0].n > TSDB_METER_ID_LEN) { setErrMsg(pCmd, msg); return TSDB_CODE_INVALID_SQL; } + */ + tDCLSQL* pDCL = pInfo->pDCLInfo; + const char* msg = "invalid configure options or values"; + + // validate the parameter names and options + if (validateLocalConfig(pDCL) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg); + return TSDB_CODE_INVALID_SQL; + } + + strncpy(pCmd->payload, pDCL->a[0].z, pDCL->a[0].n); + if (pDCL->nTokens == 2) { + pCmd->payload[pDCL->a[0].n] = ' '; // add sep + strncpy(&pCmd->payload[pDCL->a[0].n + 1], pDCL->a[1].z, pDCL->a[1].n); + } - strncpy(pCmd->payload, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n); break; } case TSQL_CREATE_NORMAL_METER: @@ -538,7 +664,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - if (setMeterID(pSql, pzTableName) != TSDB_CODE_SUCCESS) { + if (setMeterID(pSql, pzTableName, 0) != TSDB_CODE_SUCCESS) { setErrMsg(pCmd, msg); return TSDB_CODE_INVALID_SQL; } @@ -580,29 +706,29 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - int32_t ret = setMeterID(pSql, pToken); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + if (setMeterID(pSql, pToken, 0) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg); + return TSDB_CODE_INVALID_SQL; } // get meter meta from mnode STagData* pTag = (STagData*)pCmd->payload; - strncpy(pTag->name, pCmd->name, TSDB_METER_ID_LEN); + strncpy(pTag->name, pMeterMetaInfo->name, TSDB_METER_ID_LEN); tVariantList* pList = pInfo->pCreateTableInfo->usingInfo.pTagVals; - int32_t code = tscGetMeterMeta(pSql, pTag->name); + int32_t code = tscGetMeterMeta(pSql, pTag->name, 0); if (code != TSDB_CODE_SUCCESS) { return code; } - if (pSql->cmd.pMeterMeta->numOfTags != pList->nExpr) { + if (pMeterMetaInfo->pMeterMeta->numOfTags != pList->nExpr) { setErrMsg(pCmd, msg2); return TSDB_CODE_INVALID_SQL; } - /* too long tag values will be truncated automatically */ - SSchema* pTagSchema = tsGetTagSchema(pCmd->pMeterMeta); + // too long tag values will return invalid sql, not be truncated automatically + SSchema* pTagSchema = tsGetTagSchema(pMeterMetaInfo->pMeterMeta); char* tagVal = pTag->data; for (int32_t i = 0; i < pList->nExpr; ++i) { @@ -612,6 +738,14 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } + // validate the length of binary + if ((pTagSchema[i].type == TSDB_DATA_TYPE_BINARY || pTagSchema[i].type == TSDB_DATA_TYPE_NCHAR) && + pList->a[i].pVar.nLen > pTagSchema[i].bytes) { + const char* msg3 = "tag value too long"; + setErrMsg(pCmd, msg3); + return TSDB_CODE_INVALID_SQL; + } + tagVal += pTagSchema[i].bytes; } @@ -620,7 +754,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - ret = setMeterID(pSql, &pInfo->pCreateTableInfo->name); + int32_t ret = setMeterID(pSql, &pInfo->pCreateTableInfo->name, 0); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -631,8 +765,10 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSQL_CREATE_STREAM: { pCmd->command = TSDB_SQL_CREATE_TABLE; - const char* msg = "table name too long"; const char* msg1 = "invalid table name"; + const char* msg2 = "table name too long"; + const char* msg3 = "fill only available for interval query"; + const char* msg4 = "fill option not supported in stream computing"; // if sql specifies db, use it, otherwise use default db SSQLToken* pzTableName = &(pInfo->pCreateTableInfo->name); @@ -643,35 +779,38 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - SSQLToken* pSrcMeterName = &pInfo->pCreateTableInfo->pSelect->from; - if (tscValidateName(pSrcMeterName) != TSDB_CODE_SUCCESS) { + tVariantList* pSrcMeterName = pInfo->pCreateTableInfo->pSelect->from; + tVariant* pVar = &pSrcMeterName->a[0].pVar; + + SSQLToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING}; + if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) { setErrMsg(pCmd, msg1); return TSDB_CODE_INVALID_SQL; } - if (setMeterID(pSql, pSrcMeterName) != TSDB_CODE_SUCCESS) { - setErrMsg(pCmd, msg); + if (setMeterID(pSql, &srcToken, 0) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg2); return TSDB_CODE_INVALID_SQL; } - int32_t code = tscGetMeterMeta(pSql, pCmd->name); + int32_t code = tscGetMeterMeta(pSql, pMeterMetaInfo->name, 0); if (code != TSDB_CODE_SUCCESS) { return code; } - bool isMetric = UTIL_METER_IS_METRIC(pCmd); - if (buildSelectionClause(pCmd, pQuerySql->pSelection, isMetric) != TSDB_CODE_SUCCESS) { + bool isMetric = UTIL_METER_IS_METRIC(pMeterMetaInfo); + if (parseSelectClause(pCmd, pQuerySql->pSelection, isMetric) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } if (pQuerySql->pWhere != NULL) { // query condition in stream computing - if (buildQueryCond(pSql, pQuerySql->pWhere) != TSDB_CODE_SUCCESS) { + if (parseWhereClause(pSql, &pQuerySql->pWhere) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } } // set interval value - if (setIntervalClause(pCmd, pQuerySql) != TSDB_CODE_SUCCESS) { + if (parseIntervalClause(pCmd, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } else { if ((pCmd->nAggTimeInterval > 0) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd) != TSDB_CODE_SUCCESS)) { @@ -684,18 +823,23 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } // set the created table[stream] name - if (setMeterID(pSql, pzTableName) != TSDB_CODE_SUCCESS) { - setErrMsg(pCmd, msg); + if (setMeterID(pSql, pzTableName, 0) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg1); return TSDB_CODE_INVALID_SQL; } // copy sql length - tscAllocPayloadWithSize(pCmd, pQuerySql->selectToken.n + 8); + int ret = tscAllocPayload(pCmd, pQuerySql->selectToken.n + 8); + if (TSDB_CODE_SUCCESS != ret) { + const char* msg6 = "client out of memory"; + setErrMsg(pCmd, msg6); + return ret; + } strncpy(pCmd->payload, pQuerySql->selectToken.z, pQuerySql->selectToken.n); if (pQuerySql->selectToken.n > TSDB_MAX_SAVED_SQL_LEN) { - const char* msg4 = "sql too long"; // todo ADD support - setErrMsg(pCmd, msg); + const char* msg5 = "sql too long"; // todo ADD support + setErrMsg(pCmd, msg5); return TSDB_CODE_INVALID_SQL; } @@ -715,9 +859,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { */ if (pQuerySql->fillType != NULL) { if (pCmd->nAggTimeInterval == 0) { - const char* msg1 = "fill only available for interval query"; - setErrMsg(pCmd, msg1); - + setErrMsg(pCmd, msg3); return TSDB_CODE_INVALID_SQL; } @@ -725,9 +867,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) { if (!((strncmp(pItem->pVar.pz, "none", 4) == 0 && pItem->pVar.nLen == 4) || (strncmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4))) { - const char* msg2 = "fill option not supported in stream computing"; - setErrMsg(pCmd, msg2); - + setErrMsg(pCmd, msg4); return TSDB_CODE_INVALID_SQL; } } @@ -738,41 +878,79 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case TSQL_QUERY_METER: { SQuerySQL* pQuerySql = pInfo->pQueryInfo; - assert(pQuerySql != NULL); + assert(pQuerySql != NULL && pQuerySql->from->nExpr > 0); + + const char* msg0 = "invalid table name"; + const char* msg1 = "table name too long"; + const char* msg2 = "point interpolation query needs timestamp"; + const char* msg3 = "sliding value too small"; + const char* msg4 = "sliding value no larger than the interval value"; + const char* msg5 = "fill only available for interval query"; + const char* msg6 = "start(end) time of query range required or time range too large"; + const char* msg7 = "illegal number of tables in from clause"; + const char* msg8 = "too many columns in selection clause"; + const char* msg9 = "TWA query requires both the start and end time"; // too many result columns not support order by in query if (pQuerySql->pSelection->nExpr > TSDB_MAX_COLUMNS) { - const char* msg = "too many columns in selection clause"; - setErrMsg(pCmd, msg); + setErrMsg(pCmd, msg8); return TSDB_CODE_INVALID_SQL; } - if (tscValidateName(&(pQuerySql->from)) != TSDB_CODE_SUCCESS) { - const char* msg = "invalid table name"; - setErrMsg(pCmd, msg); + if (pQuerySql->from->nExpr > TSDB_MAX_JOIN_TABLE_NUM) { + setErrMsg(pCmd, msg7); return TSDB_CODE_INVALID_SQL; } - if (setMeterID(pSql, &pQuerySql->from) != TSDB_CODE_SUCCESS) { - const char* msg = "table name too long"; - setErrMsg(pCmd, msg); - return TSDB_CODE_INVALID_SQL; + // set all query tables, which are maybe more than one. + for (int32_t i = 0; i < pQuerySql->from->nExpr; ++i) { + tVariant* pTableItem = &pQuerySql->from->a[i].pVar; + + if (pTableItem->nType != TSDB_DATA_TYPE_BINARY) { + setErrMsg(pCmd, msg0); + return TSDB_CODE_INVALID_SQL; + } + + pTableItem->nLen = strdequote(pTableItem->pz); + + SSQLToken tableName = {.z = pTableItem->pz, .n = pTableItem->nLen, .type = TK_STRING}; + if (tscValidateName(&tableName) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg0); + return TSDB_CODE_INVALID_SQL; + } + + if (pCmd->numOfTables <= i) { + tscAddEmptyMeterMetaInfo(pCmd); + } + + SSQLToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; + if (setMeterID(pSql, &t, i) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg1); + return TSDB_CODE_INVALID_SQL; + } + + SMeterMetaInfo* pMeterInfo1 = tscGetMeterMetaInfo(pCmd, i); + int32_t code = tscGetMeterMeta(pSql, pMeterInfo1->name, i); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } pSql->cmd.command = TSDB_SQL_SELECT; + int32_t code = TSDB_CODE_SUCCESS; - int32_t code = tscGetMeterMeta(pSql, pCmd->name); - if (code != TSDB_CODE_SUCCESS) { - return code; + // parse the group by clause in the first place + if (parseGroupbyClause(pCmd, pQuerySql->pGroupby) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_INVALID_SQL; } - bool isMetric = UTIL_METER_IS_METRIC(pCmd); - if (buildSelectionClause(pCmd, pQuerySql->pSelection, isMetric) != TSDB_CODE_SUCCESS) { + bool isMetric = UTIL_METER_IS_METRIC(pMeterMetaInfo); + if (parseSelectClause(pCmd, pQuerySql->pSelection, isMetric) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } // set interval value - if (setIntervalClause(pCmd, pQuerySql) != TSDB_CODE_SUCCESS) { + if (parseIntervalClause(pCmd, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } else { if ((pCmd->nAggTimeInterval > 0) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd) != TSDB_CODE_SUCCESS)) { @@ -783,21 +961,18 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { // set sliding value SSQLToken* pSliding = &pQuerySql->sliding; if (pSliding->n != 0) { - // pCmd->count == 1 means sql in stream function + // TODO refactor pCmd->count == 1 means sql in stream function if (!tscEmbedded && pCmd->count == 0) { const char* msg = "not support sliding in query"; setErrMsg(pCmd, msg); return TSDB_CODE_INVALID_SQL; } - code = getTimestampInUsFromStr(pSliding->z, pSliding->n, &pCmd->nSlidingTime); - if (pCmd->pMeterMeta->precision == TSDB_TIME_PRECISION_MILLI) { + getTimestampInUsFromStr(pSliding->z, pSliding->n, &pCmd->nSlidingTime); + if (pMeterMetaInfo->pMeterMeta->precision == TSDB_TIME_PRECISION_MILLI) { pCmd->nSlidingTime /= 1000; } - const char* msg3 = "sliding value too small"; - const char* msg4 = "sliding value no larger than the interval value"; - if (pCmd->nSlidingTime < tsMinSlidingTime) { setErrMsg(pCmd, msg3); return TSDB_CODE_INVALID_SQL; @@ -809,31 +984,37 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } } - if (setGroupByClause(pCmd, pQuerySql->pGroupby) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_INVALID_SQL; - } - // set order by info - if (setOrderByClause(pCmd, pQuerySql, tsGetSchema(pCmd->pMeterMeta), pCmd->pMeterMeta->numOfColumns) != - TSDB_CODE_SUCCESS) { + if (parseOrderbyClause(pCmd, pQuerySql, tsGetSchema(pMeterMetaInfo->pMeterMeta), + pMeterMetaInfo->pMeterMeta->numOfColumns) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } // set where info if (pQuerySql->pWhere != NULL) { - if (buildQueryCond(pSql, pQuerySql->pWhere) != TSDB_CODE_SUCCESS) { + if (parseWhereClause(pSql, &pQuerySql->pWhere) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } - if (pCmd->pMeterMeta->precision == TSDB_TIME_PRECISION_MILLI) { + pQuerySql->pWhere = NULL; + + if (pMeterMetaInfo->pMeterMeta->precision == TSDB_TIME_PRECISION_MILLI) { pCmd->stime = pCmd->stime / 1000; pCmd->etime = pCmd->etime / 1000; } - } else { // set the time range + } else { // set the time rang pCmd->stime = 0; pCmd->etime = INT64_MAX; } + // user does not specified the query time window, twa is not allowed in such case. + if ((pCmd->stime == 0 || pCmd->etime == INT64_MAX || + (pCmd->etime == INT64_MAX / 1000 && pMeterMetaInfo->pMeterMeta->precision == TSDB_TIME_PRECISION_MILLI)) && + tscIsTWAQuery(pCmd)) { + setErrMsg(pCmd, msg9); + return TSDB_CODE_INVALID_SQL; + } + // no result due to invalid query time range if (pCmd->stime > pCmd->etime) { pCmd->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; @@ -841,47 +1022,56 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (!hasTimestampForPointInterpQuery(pCmd)) { - const char* msg = "point interpolation query needs timestamp"; - setErrMsg(pCmd, msg); + setErrMsg(pCmd, msg2); return TSDB_CODE_INVALID_SQL; } if (pQuerySql->fillType != NULL) { - const char* msg1 = "fill only available for interval query"; - const char* msg2 = "start(end) time of query range required or time range too large"; - if (pCmd->nAggTimeInterval == 0 && (!tscIsPointInterpQuery(pCmd))) { - setErrMsg(pCmd, msg1); + setErrMsg(pCmd, msg5); return TSDB_CODE_INVALID_SQL; } if (pCmd->nAggTimeInterval > 0) { int64_t timeRange = labs(pCmd->stime - pCmd->etime); // number of result is not greater than 10,000,000 - - // TODO define macro - if ((timeRange == 0) || (timeRange / pCmd->nAggTimeInterval) > 10000000) { - setErrMsg(pCmd, msg2); + if ((timeRange == 0) || (timeRange / pCmd->nAggTimeInterval) > MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY) { + setErrMsg(pCmd, msg6); return TSDB_CODE_INVALID_SQL; } } - int32_t ret = setFillPolicy(pCmd, pQuerySql); + int32_t ret = parseFillClause(pCmd, pQuerySql); if (ret != TSDB_CODE_SUCCESS) { return ret; } } + // in case of join query, time range is required. + if (QUERY_IS_JOIN_QUERY(pCmd->type)) { + int64_t timeRange = labs(pCmd->stime - pCmd->etime); + + if (timeRange == 0 && pCmd->stime == 0) { + setErrMsg(pCmd, msg6); + return TSDB_CODE_INVALID_SQL; + } + } + // handle the limit offset value, validate the limit pCmd->limit = pQuerySql->limit; - /* temporarily save the original limitation value */ + // temporarily save the original limitation value if ((code = parseLimitClause(pSql, pQuerySql)) != TSDB_CODE_SUCCESS) { return code; } + if ((code = doFunctionsCompatibleCheck(pSql)) != TSDB_CODE_SUCCESS) { + return code; + } + setColumnOffsetValueInResultset(pCmd); - updateTagColumnIndex(pCmd); + updateTagColumnIndex(pCmd, 0); + break; } case TSQL_INSERT: { @@ -909,17 +1099,28 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_SUCCESS; } +/* + * if the top/bottom exists, only tags columns, tbname column, and primary timestamp column + * are available. + */ static bool isTopBottomQuery(SSqlCmd* pCmd) { - if (pCmd->exprsInfo.numOfExprs != 2) { - return false; + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + int32_t functionId = tscSqlExprGet(pCmd, i)->functionId; + + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + return true; + } } - int32_t functionId = tscSqlExprGet(pCmd, 1)->sqlFuncId; - return functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TOP_DST || - functionId == TSDB_FUNC_BOTTOM_DST; + return false; } -int32_t setIntervalClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql) { +int32_t parseIntervalClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql) { + const char* msg1 = "invalid query expression"; + const char* msg2 = "interval cannot be less than 10 ms"; + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + if (pQuerySql->interval.type == 0) { return TSDB_CODE_SUCCESS; } @@ -931,7 +1132,7 @@ int32_t setIntervalClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql) { } /* revised the time precision according to the flag */ - if (pCmd->pMeterMeta->precision == TSDB_TIME_PRECISION_MILLI) { + if (pMeterMetaInfo->pMeterMeta->precision == TSDB_TIME_PRECISION_MILLI) { pCmd->nAggTimeInterval = pCmd->nAggTimeInterval / 1000; } @@ -940,8 +1141,7 @@ int32_t setIntervalClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql) { // interval cannot be less than 10 milliseconds if (pCmd->nAggTimeInterval < tsMinIntervalTime) { - const char* msg = "interval cannot be less than 10 ms"; - setErrMsg(pCmd, msg); + setErrMsg(pCmd, msg2); return TSDB_CODE_INVALID_SQL; } @@ -950,10 +1150,35 @@ int32_t setIntervalClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql) { return TSDB_CODE_SUCCESS; } - // need to add timestamp column in resultset, if interval is existed - tscSqlExprInsert(pCmd, 0, TSDB_FUNC_TS, 0, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE); + // check the invalid sql expresssion: select count(tbname)/count(tag1)/count(tag2) from super_table interval(1d); + for(int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); + if (pExpr->functionId == TSDB_FUNC_COUNT && TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { + setErrMsg(pCmd, msg1); + return TSDB_CODE_INVALID_SQL; + } + } + + // need to add timestamp column in result set, if interval is existed + uint64_t uid = tscSqlExprGet(pCmd, 0)->uid; + + int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; + for (int32_t i = 0; i < pCmd->numOfTables; ++i) { + pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, i); + if (pMeterMetaInfo->pMeterMeta->uid == uid) { + tableIndex = i; + break; + } + } + + if (tableIndex == COLUMN_INDEX_INITIAL_VAL) { + return TSDB_CODE_INVALID_SQL; + } + + SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscSqlExprInsert(pCmd, 0, TSDB_FUNC_TS, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); - SColumnList ids = {.numOfCols = 1, .ids = {0}}; + SColumnList ids = getColumnList(1, 0, PRIMARYKEY_TIMESTAMP_COL_INDEX); int32_t ret = insertResultField(pCmd, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS].aName); return ret; @@ -963,11 +1188,12 @@ int32_t setSlidingClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql) { const char* msg0 = "sliding value too small"; const char* msg1 = "sliding value no larger than the interval value"; - SSQLToken* pSliding = &pQuerySql->sliding; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + SSQLToken* pSliding = &pQuerySql->sliding; if (pSliding->n != 0) { getTimestampInUsFromStr(pSliding->z, pSliding->n, &pCmd->nSlidingTime); - if (pCmd->pMeterMeta->precision == TSDB_TIME_PRECISION_MILLI) { + if (pMeterMetaInfo->pMeterMeta->precision == TSDB_TIME_PRECISION_MILLI) { pCmd->nSlidingTime /= 1000; } @@ -985,33 +1211,31 @@ int32_t setSlidingClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql) { return TSDB_CODE_SUCCESS; } -int32_t setMeterID(SSqlObj* pSql, SSQLToken* pzTableName) { - SSqlCmd* pCmd = &(pSql->cmd); - int32_t ret = TSDB_CODE_SUCCESS; - - // clear array - memset(pCmd->name, 0, tListLen(pCmd->name)); +int32_t setMeterID(SSqlObj* pSql, SSQLToken* pzTableName, int32_t tableIndex) { const char* msg = "name too long"; + SSqlCmd* pCmd = &pSql->cmd; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, tableIndex); + int32_t code = TSDB_CODE_SUCCESS; + if (hasSpecifyDB(pzTableName)) { /* * db has been specified in sql string * so we ignore current db path */ - ret = setObjFullName(pCmd->name, getAccountId(pSql), NULL, pzTableName, NULL); - } else { - /* get current DB name first, then set it into path */ + code = setObjFullName(pMeterMetaInfo->name, getAccountId(pSql), NULL, pzTableName, NULL); + } else { // get current DB name first, then set it into path SSQLToken t = {0}; getCurrentDBName(pSql, &t); - ret = setObjFullName(pCmd->name, NULL, &t, pzTableName, NULL); + code = setObjFullName(pMeterMetaInfo->name, NULL, &t, pzTableName, NULL); } - if (ret != TSDB_CODE_SUCCESS) { + if (code != TSDB_CODE_SUCCESS) { setErrMsg(pCmd, msg); } - return ret; + return code; } static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) { @@ -1155,7 +1379,8 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { const char* msg5 = "invalid binary/nchar tag length"; const char* msg6 = "invalid data type in tags"; - SMeterMeta* pMeterMeta = pCmd->pMeterMeta; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + SMeterMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; // no more than 6 tags if (pMeterMeta->numOfTags == TSDB_MAX_TAGS) { @@ -1177,7 +1402,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { return false; } - SSchema* pTagSchema = tsGetTagSchema(pCmd->pMeterMeta); + SSchema* pTagSchema = tsGetTagSchema(pMeterMetaInfo->pMeterMeta); int32_t nLen = 0; for (int32_t i = 0; i < pMeterMeta->numOfTags; ++i) { @@ -1223,7 +1448,8 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { const char* msg5 = "invalid column name"; const char* msg6 = "invalid column length"; - SMeterMeta* pMeterMeta = pCmd->pMeterMeta; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + SMeterMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; // no more max columns if (pMeterMeta->numOfColumns >= TSDB_MAX_COLUMNS || @@ -1338,90 +1564,88 @@ static int32_t setObjFullName(char* fullName, char* account, SSQLToken* pDB, SSQ totalLen += tableName->n; } - fullName[totalLen] = 0; - if (xlen != NULL) { *xlen = totalLen; } + + if (totalLen < TSDB_METER_ID_LEN) { + fullName[totalLen] = 0; + } + return (totalLen <= TSDB_METER_ID_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_INVALID_SQL; } -static void extractColumnNameFromString(tSQLExprItem* pItem, char* tmpBuf) { +static void extractColumnNameFromString(tSQLExprItem* pItem) { if (pItem->pNode->nSQLOptr == TK_STRING) { - strdequote(pItem->pNode->val.pz); - strcpy(tmpBuf, pItem->pNode->val.pz); - - tVariantDestroy(&pItem->pNode->val); + pItem->pNode->val.nLen = strdequote(pItem->pNode->val.pz); pItem->pNode->nSQLOptr = TK_ID; SSQLToken* pIdToken = &pItem->pNode->colInfo; pIdToken->type = TK_ID; - pIdToken->z = tmpBuf; - pIdToken->n = strlen(pIdToken->z); + pIdToken->z = pItem->pNode->val.pz; + pIdToken->n = pItem->pNode->val.nLen; } } -int32_t buildSelectionClause(SSqlCmd* pCmd, tSQLExprList* pSelection, bool isMetric) { +int32_t parseSelectClause(SSqlCmd* pCmd, tSQLExprList* pSelection, bool isMetric) { assert(pSelection != NULL && pCmd != NULL); const char* msg1 = "invalid column name/illegal column type in arithmetic expression"; const char* msg2 = "functions can not be mixed up"; const char* msg3 = "not support query expression"; - const char* msg4 = "function not support in STable query"; - - SSchema* pSchema = tsGetSchema(pCmd->pMeterMeta); for (int32_t i = 0; i < pSelection->nExpr; ++i) { - int32_t outputIndex = pCmd->fieldsInfo.numOfOutputCols; - + int32_t outputIndex = pCmd->fieldsInfo.numOfOutputCols; tSQLExprItem* pItem = &pSelection->a[i]; - if (pItem->pNode->nSQLOptr == TK_ALL || pItem->pNode->nSQLOptr == TK_ID || - pItem->pNode->nSQLOptr == TK_STRING) { // project on all fields + // project on all fields + if (pItem->pNode->nSQLOptr == TK_ALL || pItem->pNode->nSQLOptr == TK_ID || pItem->pNode->nSQLOptr == TK_STRING) { + // it is actually a function, but the function name is invalid if (pItem->pNode->nSQLOptr == TK_ID && (pItem->pNode->colInfo.z == NULL && pItem->pNode->colInfo.n == 0)) { - /* it is actually a function, but the function name is invalid */ return TSDB_CODE_INVALID_SQL; } - /* if the name of column is quoted, remove it and set the right - * information for later process */ - char tmpName[TSDB_METER_NAME_LEN + 1] = {0}; - extractColumnNameFromString(pItem, tmpName); + // if the name of column is quoted, remove it and set the right information for later process + extractColumnNameFromString(pItem); - /* select * / select field_name1, field_name2 from table_name */ - int32_t ret = addProjectionExprAndResultField(pCmd, pSchema, pItem, isMetric); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + pCmd->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; + + // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 + if (addProjectionExprAndResultField(pCmd, pItem) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_INVALID_SQL; } } else if (pItem->pNode->nSQLOptr >= TK_COUNT && pItem->pNode->nSQLOptr <= TK_LAST_ROW) { - // sql function optr - /* sql function in selection clause, append sql function info in pSqlCmd structure sequentially */ - if (addExprAndResultField(pCmd, outputIndex, pItem) == -1) { + // sql function in selection clause, append sql function info in pSqlCmd structure sequentially + if (addExprAndResultField(pCmd, outputIndex, pItem) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } } else if (pItem->pNode->nSQLOptr >= TK_PLUS && pItem->pNode->nSQLOptr <= TK_REM) { - /* arithmetic function in select*/ - int32_t ret = validateArithmeticSQLExpr(pItem->pNode, pSchema, pCmd->pMeterMeta->numOfColumns); + // arithmetic function in select + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + + SColumnIdListRes columnList = {.pSchema = pSchema, .numOfCols = pMeterMetaInfo->pMeterMeta->numOfColumns}; + + int32_t ret = + validateArithmeticSQLExpr(pItem->pNode, pSchema, pMeterMetaInfo->pMeterMeta->numOfColumns, &columnList); if (ret != TSDB_CODE_SUCCESS) { setErrMsg(pCmd, msg1); return TSDB_CODE_INVALID_SQL; } - SColumnIdList ids = {0}; - ids.pSchema = pSchema; - ids.numOfCols = pCmd->pMeterMeta->numOfColumns; - char arithmeticExprStr[1024] = {0}; char* p = arithmeticExprStr; - if (buildArithmeticExprString(pItem->pNode, &p, &ids) != TSDB_CODE_SUCCESS) { + if (buildArithmeticExprString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } // expr string is set as the parameter of function - SSqlExpr* pExpr = tscSqlExprInsert(pCmd, outputIndex, TSDB_FUNC_ARITHM, 0, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - addExprParams(pExpr, arithmeticExprStr, TSDB_DATA_TYPE_BINARY, strlen(arithmeticExprStr)); + SColumnIndex index = {0}; + SSqlExpr* pExpr = tscSqlExprInsert(pCmd, outputIndex, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, + sizeof(double), sizeof(double)); + addExprParams(pExpr, arithmeticExprStr, TSDB_DATA_TYPE_BINARY, strlen(arithmeticExprStr), 0); /* todo alias name should use the original sql string */ if (pItem->aliasName != NULL) { @@ -1430,10 +1654,7 @@ int32_t buildSelectionClause(SSqlCmd* pCmd, tSQLExprList* pSelection, bool isMet strncpy(pExpr->aliasName, arithmeticExprStr, TSDB_COL_NAME_LEN); } - SColumnList idx = {.numOfCols = ids.numOfRecordedCols, .ids = {0}}; - memcpy(idx.ids, ids.ids, ids.numOfRecordedCols * sizeof(ids.ids[0])); - - insertResultField(pCmd, i, &idx, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName); + insertResultField(pCmd, i, &columnList.list, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName); } else { /* * not support such expression @@ -1454,11 +1675,12 @@ int32_t buildSelectionClause(SSqlCmd* pCmd, tSQLExprList* pSelection, bool isMet } if (isMetric) { - pCmd->metricQuery = 1; + pCmd->type |= TSDB_QUERY_TYPE_STABLE_QUERY; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); - if (onlyQueryMetricTags(pCmd)) { // local handle the metric tag query + if (tscQueryMetricTags(pCmd)) { // local handle the metric tag query pCmd->command = TSDB_SQL_RETRIEVE_TAGS; - pCmd->count = pCmd->pMeterMeta->numOfColumns; // the number of meter schema, tricky. + pCmd->count = pMeterMetaInfo->pMeterMeta->numOfColumns; // the number of meter schema, tricky. } /* @@ -1468,7 +1690,6 @@ int32_t buildSelectionClause(SSqlCmd* pCmd, tSQLExprList* pSelection, bool isMet tscTansformSQLFunctionForMetricQuery(pCmd); if (hasUnsupportFunctionsForMetricQuery(pCmd)) { - setErrMsg(pCmd, msg4); return TSDB_CODE_INVALID_SQL; } } @@ -1478,123 +1699,177 @@ int32_t buildSelectionClause(SSqlCmd* pCmd, tSQLExprList* pSelection, bool isMet int32_t insertResultField(SSqlCmd* pCmd, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName) { - for (int32_t i = 0; i < pIdList->numOfCols; ++i) { - tscColumnInfoInsert(pCmd, pIdList->ids[i]); + for (int32_t i = 0; i < pIdList->num; ++i) { + tscColumnBaseInfoInsert(pCmd, &(pIdList->ids[i])); } tscFieldInfoSetValue(&pCmd->fieldsInfo, outputIndex, type, fieldName, bytes); return TSDB_CODE_SUCCESS; } -void setProjExprForMetricQuery(SSqlCmd* pCmd, int32_t outputIndex, int32_t colIdx) { - pCmd->metricQuery = 1; - SSchema* pSchema = tsGetSchemaColIdx(pCmd->pMeterMeta, colIdx); +SSqlExpr* doAddProjectCol(SSqlCmd* pCmd, int32_t outputIndex, int32_t colIdx, int32_t tableIndex) { + SMeterMeta* pMeterMeta = tscGetMeterMetaInfo(pCmd, tableIndex)->pMeterMeta; - int16_t functionId = (int16_t)((colIdx >= pCmd->pMeterMeta->numOfColumns) ? TSDB_FUNC_TAGPRJ : // tagPrj function - TSDB_FUNC_PRJ); // colprj function + SSchema* pSchema = tsGetColumnSchema(pMeterMeta, colIdx); + int32_t numOfCols = pMeterMeta->numOfColumns; - int32_t numOfCols = pCmd->pMeterMeta->numOfColumns; + int16_t functionId = (int16_t)((colIdx >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ); - bool isTag = false; - if (colIdx >= numOfCols) { - colIdx -= numOfCols; - addRequiredTagColumn(pCmd, colIdx); - isTag = true; + if (functionId == TSDB_FUNC_TAGPRJ) { + addRequiredTagColumn(pCmd, colIdx - numOfCols, tableIndex); + pCmd->type = TSDB_QUERY_TYPE_STABLE_QUERY; + } else { + pCmd->type = TSDB_QUERY_TYPE_PROJECTION_QUERY; } - SSqlExpr* pExpr = tscSqlExprInsert(pCmd, outputIndex, functionId, colIdx, pSchema->type, pSchema->bytes); - pExpr->colInfo.isTag = isTag; + SColumnIndex index = {tableIndex, colIdx}; + SSqlExpr* pExpr = + tscSqlExprInsert(pCmd, outputIndex, functionId, &index, pSchema->type, pSchema->bytes, pSchema->bytes); + + return pExpr; } -void addRequiredTagColumn(SSqlCmd* pCmd, int32_t tagColIndex) { - if (pCmd->numOfReqTags == 0 || pCmd->tagColumnIndex[pCmd->numOfReqTags - 1] < tagColIndex) { - pCmd->tagColumnIndex[pCmd->numOfReqTags++] = tagColIndex; +void addRequiredTagColumn(SSqlCmd* pCmd, int32_t tagColIndex, int32_t tableIndex) { + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, tableIndex); + + if (pMeterMetaInfo->numOfTags == 0 || pMeterMetaInfo->tagColumnIndex[pMeterMetaInfo->numOfTags - 1] < tagColIndex) { + pMeterMetaInfo->tagColumnIndex[pMeterMetaInfo->numOfTags++] = tagColIndex; } else { // find the appropriate position - for (int32_t i = 0; i < pCmd->numOfReqTags; ++i) { - if (tagColIndex > pCmd->tagColumnIndex[i]) { + for (int32_t i = 0; i < pMeterMetaInfo->numOfTags; ++i) { + if (tagColIndex > pMeterMetaInfo->tagColumnIndex[i]) { continue; - } else if (tagColIndex == pCmd->tagColumnIndex[i]) { + } else if (tagColIndex == pMeterMetaInfo->tagColumnIndex[i]) { break; } else { - memmove(&pCmd->tagColumnIndex[i + 1], &pCmd->tagColumnIndex[i], - sizeof(pCmd->tagColumnIndex[0]) * (pCmd->numOfReqTags - i)); - pCmd->tagColumnIndex[i] = tagColIndex; + memmove(&pMeterMetaInfo->tagColumnIndex[i + 1], &pMeterMetaInfo->tagColumnIndex[i], + sizeof(pMeterMetaInfo->tagColumnIndex[0]) * (pMeterMetaInfo->numOfTags - i)); - pCmd->numOfReqTags++; + pMeterMetaInfo->tagColumnIndex[i] = tagColIndex; + + pMeterMetaInfo->numOfTags++; break; } } } // plus one means tbname - assert(tagColIndex >= -1 && tagColIndex < TSDB_MAX_TAGS && pCmd->numOfReqTags <= TSDB_MAX_TAGS + 1); + assert(tagColIndex >= -1 && tagColIndex < TSDB_MAX_TAGS && pMeterMetaInfo->numOfTags <= TSDB_MAX_TAGS + 1); } -int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SSchema* pSchema, tSQLExprItem* pItem, bool isMetric) { - int32_t startPos = pCmd->fieldsInfo.numOfOutputCols; +static void addProjectQueryCol(SSqlCmd* pCmd, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { + SSqlExpr* pExpr = doAddProjectCol(pCmd, startPos, pIndex->columnIndex, pIndex->tableIndex); - if (pItem->pNode->nSQLOptr == TK_ALL) { // project on all fields - int32_t numOfTotalColumns = 0; - SMeterMeta* pMeterMeta = pCmd->pMeterMeta; + SMeterMeta* pMeterMeta = tscGetMeterMetaInfo(pCmd, pIndex->tableIndex)->pMeterMeta; - if (isMetric) { // metric query - numOfTotalColumns = pMeterMeta->numOfColumns + pMeterMeta->numOfTags; + SSchema* pSchema = tsGetColumnSchema(pMeterMeta, pIndex->columnIndex); - for (int32_t j = 0; j < numOfTotalColumns; ++j) { - setProjExprForMetricQuery(pCmd, startPos + j, j); + char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName; - SColumnList ids = {.numOfCols = 1, .ids = {j}}; + SColumnList ids = {0}; + ids.num = 1; + ids.ids[0] = *pIndex; - // tag columns do not add to source list - if (j >= pMeterMeta->numOfColumns) { - ids.numOfCols = 0; - } - insertResultField(pCmd, startPos + j, &ids, pSchema[j].bytes, pSchema[j].type, pSchema[j].name); - } - } else { // meter query - numOfTotalColumns = pMeterMeta->numOfColumns; - for (int32_t j = 0; j < numOfTotalColumns; ++j) { - tscSqlExprInsert(pCmd, j, TSDB_FUNC_PRJ, j, pSchema[j].type, pSchema[j].bytes); + if (pIndex->columnIndex >= pMeterMeta->numOfColumns || pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + ids.num = 0; + } - SColumnList ids = {.numOfCols = 1, .ids = {j}}; - insertResultField(pCmd, startPos + j, &ids, pSchema[j].bytes, pSchema[j].type, pSchema[j].name); - } - } + insertResultField(pCmd, startPos, &ids, pExpr->resBytes, pExpr->resType, colName); +} - } else if (pItem->pNode->nSQLOptr == TK_ID) { // simple column projection query - int32_t numOfAllCols = pCmd->pMeterMeta->numOfColumns + pCmd->pMeterMeta->numOfTags; - int32_t idx = getColumnIndexByName(&pItem->pNode->colInfo, pSchema, numOfAllCols); - if (idx == -1) { - if (strncmp(pItem->pNode->colInfo.z, TSQL_TBNAME_L, 6) == 0 && pItem->pNode->colInfo.n == 6) { - SSqlExpr* pExpr = - tscSqlExprInsert(pCmd, startPos, TSDB_FUNC_TAGPRJ, -1, TSDB_DATA_TYPE_BINARY, TSDB_METER_NAME_LEN); +void tscAddSpecialColumnForSelect(SSqlCmd* pCmd, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, + SSchema* pColSchema, int16_t flag) { + SSqlExpr* pExpr = tscSqlExprInsert(pCmd, outputColIndex, functionId, pIndex, pColSchema->type, pColSchema->bytes, + pColSchema->bytes); - SColumnList ids = {.numOfCols = 1, .ids = {idx}}; - insertResultField(pCmd, startPos, &ids, TSDB_METER_NAME_LEN, TSDB_DATA_TYPE_BINARY, TSQL_TBNAME_L); + SColumnList ids = getColumnList(1, pIndex->tableIndex, pIndex->columnIndex); + if (TSDB_COL_IS_TAG(flag)) { + ids.num = 0; + } - pCmd->metricQuery = 1; - addRequiredTagColumn(pCmd, -1); - pExpr->colInfo.isTag = true; - } else { - const char* msg = "invalid column name"; - setErrMsg(pCmd, msg); - return TSDB_CODE_INVALID_SQL; + insertResultField(pCmd, outputColIndex, &ids, pColSchema->bytes, pColSchema->type, pColSchema->name); + + pExpr->colInfo.flag = flag; + if (TSDB_COL_IS_TAG(flag)) { + addRequiredTagColumn(pCmd, pIndex->columnIndex, pIndex->tableIndex); + } +} + +static int32_t doAddProjectionExprAndResultFields(SSqlCmd* pCmd, SColumnIndex* pIndex, int32_t startPos) { + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pIndex->tableIndex); + + int32_t numOfTotalColumns = 0; + SMeterMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; + SSchema* pSchema = tsGetSchema(pMeterMeta); + + if (UTIL_METER_IS_METRIC(pMeterMetaInfo)) { + numOfTotalColumns = pMeterMeta->numOfColumns + pMeterMeta->numOfTags; + } else { + numOfTotalColumns = pMeterMeta->numOfColumns; + } + + for (int32_t j = 0; j < numOfTotalColumns; ++j) { + doAddProjectCol(pCmd, startPos + j, j, pIndex->tableIndex); + + pIndex->columnIndex = j; + SColumnList ids = {0}; + ids.ids[0] = *pIndex; + + // tag columns do not add to source list + ids.num = (j >= pMeterMeta->numOfColumns) ? 0 : 1; + + insertResultField(pCmd, startPos + j, &ids, pSchema[j].bytes, pSchema[j].type, pSchema[j].name); + } + + return numOfTotalColumns; +} + +int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, tSQLExprItem* pItem) { + const char* msg0 = "invalid column name"; + const char* msg1 = "tag for table query is not allowed"; + + int32_t startPos = pCmd->fieldsInfo.numOfOutputCols; + + if (pItem->pNode->nSQLOptr == TK_ALL) { // project on all fields + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + if (getTableIndexByName(&pItem->pNode->colInfo, pCmd, &index) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_INVALID_SQL; + } + + // all meters columns are required + if (index.tableIndex == COLUMN_INDEX_INITIAL_VAL) { // all table columns are required. + for (int32_t i = 0; i < pCmd->numOfTables; ++i) { + index.tableIndex = i; + int32_t inc = doAddProjectionExprAndResultFields(pCmd, &index, startPos); + startPos += inc; } } else { - if (isMetric) { - setProjExprForMetricQuery(pCmd, startPos, idx); - } else { - tscSqlExprInsert(pCmd, startPos, TSDB_FUNC_PRJ, idx, pSchema[idx].type, pSchema[idx].bytes); - } + doAddProjectionExprAndResultFields(pCmd, &index, startPos); + } + } else if (pItem->pNode->nSQLOptr == TK_ID) { // simple column projection query + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + + if (getColumnIndexByNameEx(&pItem->pNode->colInfo, pCmd, &index) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg0); + return TSDB_CODE_INVALID_SQL; + } - char* colName = (pItem->aliasName == NULL) ? pSchema[idx].name : pItem->aliasName; - SColumnList ids = {.numOfCols = 1, .ids = {idx}}; + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + SColumnIndex index1 = {0, TSDB_TBNAME_COLUMN_INDEX}; + SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_METER_NAME_LEN}; + strcpy(colSchema.name, TSQL_TBNAME_L); + + pCmd->type = TSDB_QUERY_TYPE_STABLE_QUERY; + tscAddSpecialColumnForSelect(pCmd, startPos, TSDB_FUNC_TAGPRJ, &index1, &colSchema, true); + } else { + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); + SMeterMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; - if (idx >= pCmd->pMeterMeta->numOfColumns || idx == -1) { - ids.numOfCols = 0; + if (index.columnIndex >= pMeterMeta->numOfColumns && UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { + setErrMsg(pCmd, msg1); + return TSDB_CODE_INVALID_SQL; } - insertResultField(pCmd, startPos, &ids, pSchema[idx].bytes, pSchema[idx].type, colName); + addProjectQueryCol(pCmd, startPos, &index, pItem); } } else { return TSDB_CODE_INVALID_SQL; @@ -1604,7 +1879,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SSchema* pSchema, tSQLExp } static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SSchema* pSchema, int32_t functionID, char* aliasName, - int32_t resColIdx, int32_t idx) { + int32_t resColIdx, SColumnIndex* pColIndex) { int16_t type = 0; int16_t bytes = 0; @@ -1612,8 +1887,9 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SSchema* pSchema, int32_t const char* msg1 = "not support column types"; if (functionID == TSDB_FUNC_SPREAD) { - if (pSchema[idx].type == TSDB_DATA_TYPE_BINARY || pSchema[idx].type == TSDB_DATA_TYPE_NCHAR || - pSchema[idx].type == TSDB_DATA_TYPE_BOOL) { + if (pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_BINARY || + pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_NCHAR || + pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_BOOL) { setErrMsg(pCmd, msg1); return -1; } else { @@ -1621,93 +1897,115 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SSchema* pSchema, int32_t bytes = tDataTypeDesc[type].nSize; } } else { - type = pSchema[idx].type; - bytes = pSchema[idx].bytes; + type = pSchema[pColIndex->columnIndex].type; + bytes = pSchema[pColIndex->columnIndex].bytes; } if (aliasName != NULL) { strcpy(columnName, aliasName); } else { - getRevisedName(columnName, functionID, TSDB_COL_NAME_LEN, pSchema[idx].name); + getRevisedName(columnName, functionID, TSDB_COL_NAME_LEN, pSchema[pColIndex->columnIndex].name); } - tscSqlExprInsert(pCmd, resColIdx, functionID, idx, type, bytes); + tscSqlExprInsert(pCmd, resColIdx, functionID, pColIndex, type, bytes, bytes); - /* for point interpolation/last_row query, we need the timestamp column to be - * loaded */ + // for point interpolation/last_row query, we need the timestamp column to be loaded + SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; if (functionID == TSDB_FUNC_INTERP || functionID == TSDB_FUNC_LAST_ROW) { - tscColumnInfoInsert(pCmd, PRIMARYKEY_TIMESTAMP_COL_INDEX); + tscColumnBaseInfoInsert(pCmd, &index); } - SColumnList ids = {.numOfCols = 1, .ids = {idx}}; + SColumnList ids = getColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex); insertResultField(pCmd, resColIdx, &ids, bytes, type, columnName); return TSDB_CODE_SUCCESS; } int32_t addExprAndResultField(SSqlCmd* pCmd, int32_t colIdx, tSQLExprItem* pItem) { - int32_t optr = pItem->pNode->nSQLOptr; - SSchema* pSchema = tsGetSchema(pCmd->pMeterMeta); - int32_t numOfAddedColumn = 1; + SMeterMetaInfo* pMeterMetaInfo = NULL; + int32_t optr = pItem->pNode->nSQLOptr; + + int32_t numOfAddedColumn = 1; - const char* msg = "invalid parameters"; const char* msg1 = "not support column types"; + const char* msg2 = "invalid parameters"; const char* msg3 = "illegal column name"; + const char* msg4 = "invalid table name"; const char* msg5 = "parameter is out of range [0, 100]"; + const char* msg6 = "function applied to tags not allowed"; switch (optr) { case TK_COUNT: { if (pItem->pNode->pParam != NULL && pItem->pNode->pParam->nExpr != 1) { /* more than one parameter for count() function */ - setErrMsg(pCmd, msg); - return -1; + setErrMsg(pCmd, msg2); + return TSDB_CODE_INVALID_SQL; } int16_t functionID = 0; if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) { - return -1; + return TSDB_CODE_INVALID_SQL; } - int32_t columnId = 0; + SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (pItem->pNode->pParam != NULL) { SSQLToken* pToken = &pItem->pNode->pParam->a[0].pNode->colInfo; if (pToken->z == NULL || pToken->n == 0) { setErrMsg(pCmd, msg3); - return -1; + return TSDB_CODE_INVALID_SQL; } - /* count the number of meters created according to the metric */ - if (strncmp(pToken->z, "tbname", 6) == 0 && pToken->n == 6) { - tscSqlExprInsert(pCmd, colIdx, functionID, -1, TSDB_DATA_TYPE_BIGINT, - tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize); + tSQLExprItem* pParamElem = &pItem->pNode->pParam->a[0]; + if (pParamElem->pNode->nSQLOptr == TK_ALL) { + // select table.* + // check if the table name is valid or not + SSQLToken tmpToken = pParamElem->pNode->colInfo; + + if (getTableIndexByName(&tmpToken, pCmd, &index) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg4); + return TSDB_CODE_INVALID_SQL; + } + + index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; + tscSqlExprInsert(pCmd, colIdx, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } else { - columnId = getColumnIndexByName(pToken, pSchema, pCmd->pMeterMeta->numOfColumns); - if (columnId < 0) { // invalid column name + // count the number of meters created according to the metric + if (getColumnIndexByNameEx(pToken, pCmd, &index) != TSDB_CODE_SUCCESS) { setErrMsg(pCmd, msg3); - return -1; + return TSDB_CODE_INVALID_SQL; + } + + pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); + + // count tag is equalled to count(tbname) + if (index.columnIndex >= pMeterMetaInfo->pMeterMeta->numOfColumns) { + index.columnIndex = TSDB_TBNAME_COLUMN_INDEX; } - tscSqlExprInsert(pCmd, colIdx, functionID, columnId, TSDB_DATA_TYPE_BIGINT, - tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize); + int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; + tscSqlExprInsert(pCmd, colIdx, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } - } else { - /* count(*) is equalled to count(primary_timestamp_key) */ - tscSqlExprInsert(pCmd, colIdx, functionID, PRIMARYKEY_TIMESTAMP_COL_INDEX, TSDB_DATA_TYPE_BIGINT, - tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize); + } else { // count(*) is equalled to count(primary_timestamp_key) + index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + + int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; + tscSqlExprInsert(pCmd, colIdx, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } char columnName[TSDB_COL_NAME_LEN] = {0}; getColumnName(pItem, columnName, TSDB_COL_NAME_LEN); // count always use the primary timestamp key column, which is 0. - SColumnList ids = {.numOfCols = 1, .ids = {columnId}}; + SColumnList ids = getColumnList(1, index.tableIndex, index.columnIndex); + insertResultField(pCmd, colIdx, &ids, sizeof(int64_t), TSDB_DATA_TYPE_BIGINT, columnName); - return numOfAddedColumn; + return TSDB_CODE_SUCCESS; } case TK_SUM: case TK_AVG: - case TK_WAVG: + case TK_TWA: case TK_MIN: case TK_MAX: case TK_DIFF: @@ -1717,27 +2015,30 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, int32_t colIdx, tSQLExprItem* pItem if (pItem->pNode->pParam == NULL || (optr != TK_LEASTSQUARES && pItem->pNode->pParam->nExpr != 1) || (optr == TK_LEASTSQUARES && pItem->pNode->pParam->nExpr != 3)) { /* no parameters or more than one parameter for function */ - setErrMsg(pCmd, msg); - return -1; + setErrMsg(pCmd, msg2); + return TSDB_CODE_INVALID_SQL; } tSQLExprItem* pParamElem = &(pItem->pNode->pParam->a[0]); if (pParamElem->pNode->nSQLOptr != TK_ALL && pParamElem->pNode->nSQLOptr != TK_ID) { - setErrMsg(pCmd, msg); - return -1; + setErrMsg(pCmd, msg2); + return TSDB_CODE_INVALID_SQL; } - int32_t idx = getColumnIndexByName(&pParamElem->pNode->colInfo, pSchema, pCmd->pMeterMeta->numOfColumns); - if (idx < 0) { // invalid column name + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByNameEx(&pParamElem->pNode->colInfo, pCmd, &index) != TSDB_CODE_SUCCESS) { setErrMsg(pCmd, msg3); - return -1; + return TSDB_CODE_INVALID_SQL; } // 2. check if sql function can be applied on this column data type - int16_t colType = pSchema[idx].type; + pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); + SSchema* pSchema = tsGetColumnSchema(pMeterMetaInfo->pMeterMeta, index.columnIndex); + int16_t colType = pSchema->type; + if (colType == TSDB_DATA_TYPE_BOOL || colType >= TSDB_DATA_TYPE_BINARY) { setErrMsg(pCmd, msg1); - return -1; + return TSDB_CODE_INVALID_SQL; } char columnName[TSDB_COL_NAME_LEN] = {0}; @@ -1745,24 +2046,35 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, int32_t colIdx, tSQLExprItem* pItem int16_t resultType = 0; int16_t resultSize = 0; + int16_t intermediateResSize = 0; int16_t functionID = 0; if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) { - return -1; + return TSDB_CODE_INVALID_SQL; } - getResultInfo(pSchema[idx].type, pSchema[idx].bytes, functionID, 0, &resultType, &resultSize); + if (getResultDataInfo(pSchema->type, pSchema->bytes, functionID, 0, &resultType, &resultSize, + &intermediateResSize, 0, false) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_INVALID_SQL; + } + // set the first column ts for diff query if (optr == TK_DIFF) { - // set the first column ts for diff query colIdx += 1; - tscSqlExprInsert(pCmd, 0, TSDB_FUNC_TS_DUMMY, 0, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE); + SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; + tscSqlExprInsert(pCmd, 0, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); - SColumnList ids = {.numOfCols = 1, .ids = {0}}; + SColumnList ids = getColumnList(1, 0, 0); insertResultField(pCmd, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].aName); } - SSqlExpr* pExpr = tscSqlExprInsert(pCmd, colIdx, functionID, idx, resultType, resultSize); + // functions can not be applied to tags + if (index.columnIndex >= pMeterMetaInfo->pMeterMeta->numOfColumns) { + setErrMsg(pCmd, msg6); + return TSDB_CODE_INVALID_SQL; + } + + SSqlExpr* pExpr = tscSqlExprInsert(pCmd, colIdx, functionID, &index, resultType, resultSize, resultSize); if (optr == TK_LEASTSQUARES) { /* set the leastsquares parameters */ @@ -1771,20 +2083,23 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, int32_t colIdx, tSQLExprItem* pItem return TSDB_CODE_INVALID_SQL; } - addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); + addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, DOUBLE_BYTES, 0); memset(val, 0, tListLen(val)); if (tVariantDump(&pParamElem[2].pNode->val, val, TSDB_DATA_TYPE_DOUBLE) < 0) { return TSDB_CODE_INVALID_SQL; } - addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); + addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0); } - SColumnList ids = {.numOfCols = 1, .ids = {idx}}; + SColumnList ids = {0}; + ids.num = 1; + ids.ids[0] = index; + insertResultField(pCmd, colIdx, &ids, pExpr->resBytes, pExpr->resType, columnName); - return numOfAddedColumn; + return TSDB_CODE_SUCCESS; } case TK_FIRST: case TK_LAST: @@ -1799,7 +2114,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, int32_t colIdx, tSQLExprItem* pItem if (!requireAllFields) { if (pItem->pNode->pParam->nExpr < 1) { setErrMsg(pCmd, msg3); - return -1; + return TSDB_CODE_INVALID_SQL; } /* in first/last function, multiple columns can be add to resultset */ @@ -1808,28 +2123,70 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, int32_t colIdx, tSQLExprItem* pItem tSQLExprItem* pParamElem = &(pItem->pNode->pParam->a[i]); if (pParamElem->pNode->nSQLOptr != TK_ALL && pParamElem->pNode->nSQLOptr != TK_ID) { setErrMsg(pCmd, msg3); - return -1; + return TSDB_CODE_INVALID_SQL; } - int32_t idx = getColumnIndexByName(&pParamElem->pNode->colInfo, pSchema, pCmd->pMeterMeta->numOfColumns); - if (idx == -1) { - return -1; - } + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + + if (pParamElem->pNode->nSQLOptr == TK_ALL) { + // select table.* + SSQLToken tmpToken = pParamElem->pNode->colInfo; + + if (getTableIndexByName(&tmpToken, pCmd, &index) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg4); + return TSDB_CODE_INVALID_SQL; + } + + pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); + SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + + for (int32_t j = 0; j < pMeterMetaInfo->pMeterMeta->numOfColumns; ++j) { + index.columnIndex = j; + if (setExprInfoForFunctions(pCmd, pSchema, functionID, pItem->aliasName, colIdx++, &index) != 0) { + return TSDB_CODE_INVALID_SQL; + } + } - if (setExprInfoForFunctions(pCmd, pSchema, functionID, pItem->aliasName, colIdx + i, idx) != 0) { - return -1; + } else { + if (getColumnIndexByNameEx(&pParamElem->pNode->colInfo, pCmd, &index) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg3); + return TSDB_CODE_INVALID_SQL; + } + + pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); + SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + + // functions can not be applied to tags + if (index.columnIndex >= pMeterMetaInfo->pMeterMeta->numOfColumns) { + setErrMsg(pCmd, msg6); + return TSDB_CODE_INVALID_SQL; + } + + if (setExprInfoForFunctions(pCmd, pSchema, functionID, pItem->aliasName, colIdx + i, &index) != 0) { + return TSDB_CODE_INVALID_SQL; + } } } - return pItem->pNode->pParam->nExpr; - } else { - for (int32_t i = 0; i < pCmd->pMeterMeta->numOfColumns; ++i) { - if (setExprInfoForFunctions(pCmd, pSchema, functionID, pItem->aliasName, colIdx + i, i) != 0) { - return -1; + return TSDB_CODE_SUCCESS; + } else { // select * from xxx + int32_t numOfFields = 0; + + for (int32_t j = 0; j < pCmd->numOfTables; ++j) { + pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, j); + SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + + for (int32_t i = 0; i < pMeterMetaInfo->pMeterMeta->numOfColumns; ++i) { + SColumnIndex index = {.tableIndex = j, .columnIndex = i}; + if (setExprInfoForFunctions(pCmd, pSchema, functionID, pItem->aliasName, colIdx + i + j, &index) != 0) { + return TSDB_CODE_INVALID_SQL; + } } + + numOfFields += pMeterMetaInfo->pMeterMeta->numOfColumns; } - return pCmd->pMeterMeta->numOfColumns; + return TSDB_CODE_SUCCESS; } } case TK_TOP: @@ -1839,49 +2196,60 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, int32_t colIdx, tSQLExprItem* pItem // 1. valid the number of parameters if (pItem->pNode->pParam == NULL || pItem->pNode->pParam->nExpr != 2) { /* no parameters or more than one parameter for function */ - setErrMsg(pCmd, msg); - return -1; + setErrMsg(pCmd, msg2); + return TSDB_CODE_INVALID_SQL; } tSQLExprItem* pParamElem = &(pItem->pNode->pParam->a[0]); if (pParamElem->pNode->nSQLOptr != TK_ID) { - setErrMsg(pCmd, msg); + setErrMsg(pCmd, msg2); + return TSDB_CODE_INVALID_SQL; } char columnName[TSDB_COL_NAME_LEN] = {0}; getColumnName(pItem, columnName, TSDB_COL_NAME_LEN); - int32_t idx = getColumnIndexByName(&pParamElem->pNode->colInfo, pSchema, pCmd->pMeterMeta->numOfColumns); - if (idx == -1) { - return -1; + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByNameEx(&pParamElem->pNode->colInfo, pCmd, &index) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg3); + return TSDB_CODE_INVALID_SQL; + } + + pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); + SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + + // functions can not be applied to tags + if (index.columnIndex >= pMeterMetaInfo->pMeterMeta->numOfColumns) { + setErrMsg(pCmd, msg6); + return TSDB_CODE_INVALID_SQL; } // 2. valid the column type - int16_t colType = pSchema[idx].type; + int16_t colType = pSchema[index.columnIndex].type; if (colType == TSDB_DATA_TYPE_BOOL || colType >= TSDB_DATA_TYPE_BINARY) { setErrMsg(pCmd, msg1); - return -1; + return TSDB_CODE_INVALID_SQL; } // 3. valid the parameters if (pParamElem[1].pNode->nSQLOptr == TK_ID) { - setErrMsg(pCmd, msg); - return -1; + setErrMsg(pCmd, msg2); + return TSDB_CODE_INVALID_SQL; } tVariant* pVariant = &pParamElem[1].pNode->val; - int8_t resultType = pSchema[idx].type; - int16_t resultSize = pSchema[idx].bytes; + int8_t resultType = pSchema[index.columnIndex].type; + int16_t resultSize = pSchema[index.columnIndex].bytes; char val[8] = {0}; if (optr == TK_PERCENTILE || optr == TK_APERCENTILE) { tVariantDump(pVariant, val, TSDB_DATA_TYPE_DOUBLE); double dp = *((double*)val); - if (dp < 0 || dp > 100) { // todo use macro + if (dp < 0 || dp > TOP_BOTTOM_QUERY_LIMIT) { setErrMsg(pCmd, msg5); - return -1; + return TSDB_CODE_INVALID_SQL; } resultSize = sizeof(double); @@ -1894,44 +2262,65 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, int32_t colIdx, tSQLExprItem* pItem */ int16_t functionId = 0; if (changeFunctionID(optr, &functionId) != TSDB_CODE_SUCCESS) { - return -1; + return TSDB_CODE_INVALID_SQL; } - SSqlExpr* pExpr = tscSqlExprInsert(pCmd, colIdx, functionId, idx, resultType, resultSize); - addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); + SSqlExpr* pExpr = tscSqlExprInsert(pCmd, colIdx, functionId, &index, resultType, resultSize, resultSize); + addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0); } else { tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT); int64_t nTop = *((int32_t*)val); if (nTop <= 0 || nTop > 100) { // todo use macro - return -1; + setErrMsg(pCmd, msg5); + return TSDB_CODE_INVALID_SQL; } int16_t functionId = 0; if (changeFunctionID(optr, &functionId) != TSDB_CODE_SUCCESS) { - return -1; + return TSDB_CODE_INVALID_SQL; } + // set the first column ts for top/bottom query - tscSqlExprInsert(pCmd, 0, TSDB_FUNC_TS, 0, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE); - SColumnList ids = {.numOfCols = 1, .ids = {0}}; - insertResultField(pCmd, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS].aName); + SColumnIndex index1 = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscSqlExprInsert(pCmd, 0, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); + + const int32_t TS_COLUMN_INDEX = 0; + SColumnList ids = getColumnList(1, 0, TS_COLUMN_INDEX); + insertResultField(pCmd, TS_COLUMN_INDEX, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, + aAggs[TSDB_FUNC_TS].aName); colIdx += 1; // the first column is ts numOfAddedColumn += 1; - SSqlExpr* pExpr = tscSqlExprInsert(pCmd, colIdx, functionId, idx, resultType, resultSize); - addExprParams(pExpr, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t)); + SSqlExpr* pExpr = tscSqlExprInsert(pCmd, colIdx, functionId, &index, resultType, resultSize, resultSize); + addExprParams(pExpr, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0); } - SColumnList ids = {.numOfCols = 1, .ids = {idx}}; + SColumnList ids = getColumnList(1, 0, index.columnIndex); insertResultField(pCmd, colIdx, &ids, resultSize, resultType, columnName); - return numOfAddedColumn; + + return TSDB_CODE_SUCCESS; } default: - return -1; + return TSDB_CODE_INVALID_SQL; } } +// todo refactor +static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex) { + assert(num == 1 && columnIndex >= -1 && tableIndex >= 0); + + SColumnList columnList = {0}; + columnList.num = num; + + int32_t index = num - 1; + columnList.ids[index].tableIndex = tableIndex; + columnList.ids[index].columnIndex = columnIndex; + + return columnList; +} + void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength) { if (pItem->aliasName != NULL) { strncpy(resultFieldName, pItem->aliasName, nameLength); @@ -1945,26 +2334,132 @@ void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, c snprintf(resultFieldName, maxLen, "%s(%s)", aAggs[functionId].aName, columnName); } -int32_t getColumnIndexByName(SSQLToken* pToken, SSchema* pSchema, int32_t numOfCols) { - if (pToken->z == NULL || pToken->n == 0) { - return -1; +static bool isTablenameToken(SSQLToken* token) { + SSQLToken tmpToken = *token; + SSQLToken tableToken = {0}; + + extractTableNameFromToken(&tmpToken, &tableToken); + + return (strncasecmp(TSQL_TBNAME_L, tmpToken.z, tmpToken.n) == 0 && tmpToken.n == strlen(TSQL_TBNAME_L)); +} + +static int16_t doGetColumnIndex(SSqlCmd* pCmd, int32_t index, SSQLToken* pToken) { + SMeterMeta* pMeterMeta = tscGetMeterMetaInfo(pCmd, index)->pMeterMeta; + + int32_t numOfCols = pMeterMeta->numOfColumns + pMeterMeta->numOfTags; + SSchema* pSchema = tsGetSchema(pMeterMeta); + + int16_t columnIndex = COLUMN_INDEX_INITIAL_VAL; + + for (int16_t i = 0; i < numOfCols; ++i) { + if (pToken->n != strlen(pSchema[i].name)) { + continue; + } + + if (strncasecmp(pSchema[i].name, pToken->z, pToken->n) == 0) { + columnIndex = i; + } + } + + return columnIndex; +} + +int32_t doGetColumnIndexByName(SSQLToken* pToken, SSqlCmd* pCmd, SColumnIndex* pIndex) { + const char* msg0 = "ambiguous column name"; + const char* msg1 = "invalid column name"; + + if (isTablenameToken(pToken)) { + pIndex->columnIndex = TSDB_TBNAME_COLUMN_INDEX; + } else if (strncasecmp(pToken->z, DEFAULT_PRIMARY_TIMESTAMP_COL_NAME, pToken->n) == 0) { + pIndex->columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX; + } else { + // not specify the table name, try to locate the table index by column name + if (pIndex->tableIndex == COLUMN_INDEX_INITIAL_VAL) { + for (int16_t i = 0; i < pCmd->numOfTables; ++i) { + int16_t colIndex = doGetColumnIndex(pCmd, i, pToken); + + if (colIndex != COLUMN_INDEX_INITIAL_VAL) { + if (pIndex->columnIndex != COLUMN_INDEX_INITIAL_VAL) { + setErrMsg(pCmd, msg0); + return TSDB_CODE_INVALID_SQL; + } else { + pIndex->tableIndex = i; + pIndex->columnIndex = colIndex; + } + } + } + } else { // table index is valid, get the column index + int16_t colIndex = doGetColumnIndex(pCmd, pIndex->tableIndex, pToken); + if (colIndex != COLUMN_INDEX_INITIAL_VAL) { + pIndex->columnIndex = colIndex; + } + } + + if (pIndex->columnIndex == COLUMN_INDEX_INITIAL_VAL) { + setErrMsg(pCmd, msg1); + return TSDB_CODE_INVALID_SQL; + } + } + + if (COLUMN_INDEX_VALIDE(*pIndex)) { + return TSDB_CODE_SUCCESS; + } else { + return TSDB_CODE_INVALID_SQL; + } +} + +static int32_t getMeterIndex(SSQLToken* pTableToken, SSqlCmd* pCmd, SColumnIndex* pIndex) { + if (pTableToken->n == 0) { // only one table and no table name prefix in column name + if (pCmd->numOfTables == 1) { + pIndex->tableIndex = 0; + } + + return TSDB_CODE_SUCCESS; + } + + pIndex->tableIndex = COLUMN_INDEX_INITIAL_VAL; + char tableName[TSDB_METER_ID_LEN + 1] = {0}; + + for (int32_t i = 0; i < pCmd->numOfTables; ++i) { + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, i); + extractMeterName(pMeterMetaInfo->name, tableName); + + if (strncasecmp(tableName, pTableToken->z, pTableToken->n) == 0 && strlen(tableName) == pTableToken->n) { + pIndex->tableIndex = i; + break; + } + } + + if (pIndex->tableIndex < 0) { + return TSDB_CODE_INVALID_SQL; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t getTableIndexByName(SSQLToken* pToken, SSqlCmd* pCmd, SColumnIndex* pIndex) { + SSQLToken tableToken = {0}; + extractTableNameFromToken(pToken, &tableToken); + + if (getMeterIndex(&tableToken, pCmd, pIndex) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_INVALID_SQL; } - char* r = strnchr(pToken->z, '.', pToken->n, false); - if (r != NULL) { - r += 1; + return TSDB_CODE_SUCCESS; +} - pToken->n -= (r - pToken->z); - pToken->z = r; +int32_t getColumnIndexByNameEx(SSQLToken* pToken, SSqlCmd* pCmd, SColumnIndex* pIndex) { + if (pCmd->pMeterInfo == NULL || pCmd->numOfTables == 0) { + return TSDB_CODE_INVALID_SQL; } - if (strncasecmp(pToken->z, "_c0", pToken->n) == 0) return 0; - for (int32_t i = 0; i < numOfCols; ++i) { - if (pToken->n != strlen(pSchema[i].name)) continue; - if (strncasecmp(pSchema[i].name, pToken->z, pToken->n) == 0) return i; + SSQLToken tmpToken = *pToken; + + if (getTableIndexByName(&tmpToken, pCmd, pIndex) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_INVALID_SQL; } - return -1; + return doGetColumnIndexByName(&tmpToken, pCmd, pIndex); } int32_t changeFunctionID(int32_t optr, int16_t* functionId) { @@ -2014,8 +2509,8 @@ int32_t changeFunctionID(int32_t optr, int16_t* functionId) { case TK_SPREAD: *functionId = TSDB_FUNC_SPREAD; break; - case TK_WAVG: - *functionId = TSDB_FUNC_WAVG; + case TK_TWA: + *functionId = TSDB_FUNC_TWA; break; case TK_INTERP: *functionId = TSDB_FUNC_INTERP; @@ -2032,7 +2527,9 @@ int32_t changeFunctionID(int32_t optr, int16_t* functionId) { // TODO support like for showing metrics, there are show meters with like ops int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { - SSqlCmd* pCmd = &pSql->cmd; + SSqlCmd* pCmd = &pSql->cmd; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + pCmd->command = TSDB_SQL_SHOW; int8_t type = pInfo->sqlType; @@ -2042,32 +2539,50 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { switch (type) { case SHOW_VGROUPS: - pCmd->type = TSDB_MGMT_TABLE_VGROUP; + pCmd->showType = TSDB_MGMT_TABLE_VGROUP; break; case SHOW_TABLES: - pCmd->type = TSDB_MGMT_TABLE_TABLE; + pCmd->showType = TSDB_MGMT_TABLE_TABLE; break; case SHOW_STABLES: - pCmd->type = TSDB_MGMT_TABLE_METRIC; + pCmd->showType = TSDB_MGMT_TABLE_METRIC; break; case SHOW_DATABASES: - pCmd->type = TSDB_MGMT_TABLE_DB; + pCmd->showType = TSDB_MGMT_TABLE_DB; + break; + case SHOW_MNODES: + pCmd->showType = TSDB_MGMT_TABLE_MNODE; break; case SHOW_DNODES: - pCmd->type = TSDB_MGMT_TABLE_PNODE; + pCmd->showType = TSDB_MGMT_TABLE_PNODE; + break; + case SHOW_ACCOUNTS: + pCmd->showType = TSDB_MGMT_TABLE_ACCT; break; case SHOW_USERS: - pCmd->type = TSDB_MGMT_TABLE_USER; + pCmd->showType = TSDB_MGMT_TABLE_USER; + break; + case SHOW_MODULES: + pCmd->showType = TSDB_MGMT_TABLE_MODULE; break; case SHOW_CONNECTIONS: - pCmd->type = TSDB_MGMT_TABLE_CONNS; + pCmd->showType = TSDB_MGMT_TABLE_CONNS; break; case SHOW_QUERIES: - pCmd->type = TSDB_MGMT_TABLE_QUERIES; + pCmd->showType = TSDB_MGMT_TABLE_QUERIES; + break; + case SHOW_SCORES: + pCmd->showType = TSDB_MGMT_TABLE_SCORES; + break; + case SHOW_GRANTS: + pCmd->showType = TSDB_MGMT_TABLE_GRANTS; break; case SHOW_STREAMS: - pCmd->type = TSDB_MGMT_TABLE_STREAMS; + pCmd->showType = TSDB_MGMT_TABLE_STREAMS; + break; + case SHOW_CONFIGS: + pCmd->showType = TSDB_MGMT_TABLE_CONFIGS; break; default: return TSDB_CODE_INVALID_SQL; @@ -2094,13 +2609,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { int32_t ret = 0; if (pDbPrefixToken->n > 0) { // has db prefix - if (pCmd->tagCond.allocSize < TSDB_MAX_TAGS_LEN) { - pCmd->tagCond.pData = realloc(pCmd->tagCond.pData, TSDB_MAX_TAGS_LEN); - pCmd->tagCond.allocSize = TSDB_MAX_TAGS_LEN; - } - ret = setObjFullName(pCmd->tagCond.pData, getAccountId(pSql), pDbPrefixToken, NULL, &pCmd->tagCond.len); - } else { - ret = setObjFullName(pCmd->name, getAccountId(pSql), NULL, NULL, NULL); + ret = setObjFullName(pMeterMetaInfo->name, getAccountId(pSql), pDbPrefixToken, NULL, NULL); } if (ret != TSDB_CODE_SUCCESS) { @@ -2110,9 +2619,9 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (type != SHOW_VGROUPS && pInfo->pDCLInfo->nTokens == 2) { // set the like conds for show tables SSQLToken* likeToken = &pInfo->pDCLInfo->a[1]; + strncpy(pCmd->payload, likeToken->z, likeToken->n); - strdequote(pCmd->payload); - pCmd->payloadLen = strlen(pCmd->payload); + pCmd->payloadLen = strdequote(pCmd->payload); if (pCmd->payloadLen > TSDB_METER_NAME_LEN) { setErrMsg(pCmd, msg2); @@ -2183,34 +2692,46 @@ bool validateIpAddress(char* ip) { return (ipAddr != 0) && (ipAddr != 0xffffffff); } -void tscTansformSQLFunctionForMetricQuery(SSqlCmd* pCmd) { - if (pCmd->pMeterMeta == NULL || !UTIL_METER_IS_METRIC(pCmd)) { - return; +int32_t tscTansformSQLFunctionForMetricQuery(SSqlCmd* pCmd) { + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + + if (pMeterMetaInfo->pMeterMeta == NULL || !UTIL_METER_IS_METRIC(pMeterMetaInfo)) { + return TSDB_CODE_INVALID_SQL; } - assert(pCmd->pMeterMeta->numOfTags >= 0); + assert(pMeterMetaInfo->pMeterMeta->numOfTags >= 0); int16_t bytes = 0; int16_t type = 0; + int16_t intermediateBytes = 0; for (int32_t k = 0; k < pCmd->fieldsInfo.numOfOutputCols; ++k) { SSqlExpr* pExpr = tscSqlExprGet(pCmd, k); TAOS_FIELD* pField = tscFieldInfoGetField(pCmd, k); - int16_t functionId = aAggs[pExpr->sqlFuncId].stableFuncId; + int16_t functionId = aAggs[pExpr->functionId].stableFuncId; + + if ((functionId >= TSDB_FUNC_SUM && functionId <= TSDB_FUNC_TWA) || + (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST)) { + if (getResultDataInfo(pField->type, pField->bytes, functionId, pExpr->param[0].i64Key, &type, &bytes, + &intermediateBytes, 0, true) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_INVALID_SQL; + } - if (functionId >= TSDB_FUNC_SUM_DST && functionId <= TSDB_FUNC_APERCT_DST) { - getResultInfo(pField->type, pField->bytes, functionId, pExpr->param[0].i64Key, &type, &bytes); tscSqlExprUpdate(pCmd, k, functionId, pExpr->colInfo.colIdx, TSDB_DATA_TYPE_BINARY, bytes); + // todo refactor + pExpr->interResBytes = intermediateBytes; } } - tscFieldInfoRenewOffsetForInterResult(pCmd); + tscFieldInfoUpdateOffset(pCmd); + return TSDB_CODE_SUCCESS; } /* transfer the field-info back to original input format */ void tscRestoreSQLFunctionForMetricQuery(SSqlCmd* pCmd) { - if (!UTIL_METER_IS_METRIC(pCmd)) { + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + if (!UTIL_METER_IS_METRIC(pMeterMetaInfo)) { return; } @@ -2218,67 +2739,139 @@ void tscRestoreSQLFunctionForMetricQuery(SSqlCmd* pCmd) { SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); TAOS_FIELD* pField = tscFieldInfoGetField(pCmd, i); - if (pExpr->sqlFuncId >= TSDB_FUNC_SUM_DST && pExpr->sqlFuncId <= TSDB_FUNC_WAVG_DST) { + if ((pExpr->functionId >= TSDB_FUNC_FIRST_DST && pExpr->functionId <= TSDB_FUNC_LAST_DST) || + (pExpr->functionId >= TSDB_FUNC_SUM && pExpr->functionId <= TSDB_FUNC_MAX)) { pExpr->resBytes = pField->bytes; pExpr->resType = pField->type; } } } -bool onlyQueryMetricTags(SSqlCmd* pCmd) { - assert(pCmd->metricQuery == 1); +bool hasUnsupportFunctionsForMetricQuery(SSqlCmd* pCmd) { + const char* msg1 = "TWA not allowed to apply to super table directly"; + const char* msg2 = "functions not supported for super table"; + const char* msg3 = "TWA only support group by tbname for super table query"; + // filter sql function not supported by metric query yet. for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { - if (tscSqlExprGet(pCmd, i)->sqlFuncId != TSDB_FUNC_TAGPRJ) { // 18 == "tagprj" function - return false; + int32_t functionId = tscSqlExprGet(pCmd, i)->functionId; + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_METRIC) == 0) { + return true; } } - return true; -} + if (tscIsTWAQuery(pCmd)) { + if (pCmd->groupbyExpr.numOfGroupCols == 0) { + setErrMsg(pCmd, msg1); + return true; + } -bool hasUnsupportFunctionsForMetricQuery(SSqlCmd* pCmd) { - // filter sql function not supported by metric query yet. - for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { - int32_t functionId = tscSqlExprGet(pCmd, i)->sqlFuncId; - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_METRIC) == 0) { + if (pCmd->groupbyExpr.numOfGroupCols != 1 || pCmd->groupbyExpr.columnInfo[0].colIdx != TSDB_TBNAME_COLUMN_INDEX) { + setErrMsg(pCmd, msg3); return true; } } + return false; } static bool functionCompatibleCheck(SSqlCmd* pCmd) { + const char* msg1 = "column on select clause not allowed"; + int32_t startIdx = 0; - int32_t functionID = tscSqlExprGet(pCmd, startIdx)->sqlFuncId; + int32_t functionID = tscSqlExprGet(pCmd, startIdx)->functionId; + + // ts function can be simultaneously used with any other functions. if (functionID == TSDB_FUNC_TS || functionID == TSDB_FUNC_TS_DUMMY) { - startIdx++; // ts function can be simultaneously used with any other - // functions. + startIdx++; } - int32_t nRetCount = funcCompatList[tscSqlExprGet(pCmd, startIdx)->sqlFuncId]; + int32_t factor = funcCompatDefList[tscSqlExprGet(pCmd, startIdx)->functionId]; // diff function cannot be executed with other function // arithmetic function can be executed with other arithmetic functions for (int32_t i = startIdx + 1; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); - if (funcCompatList[pExpr->sqlFuncId] != nRetCount) { + int16_t functionId = tscSqlExprGet(pCmd, i)->functionId; + if (functionId == TSDB_FUNC_TAGPRJ || + functionId == TSDB_FUNC_TAG || + functionId == TSDB_FUNC_TS) { + continue; + } + + if (funcCompatDefList[functionId] != factor) { return false; } } + // additional check for select aggfuntion(column), column1 from table_name group by(column1); + if ((pCmd->type & TSDB_QUERY_TYPE_PROJECTION_QUERY) == TSDB_QUERY_TYPE_PROJECTION_QUERY) { + bool isAggFunc = false; + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + int16_t functionId = tscSqlExprGet(pCmd, i)->functionId; + + if (functionId == TSDB_FUNC_PRJ || + functionId == TSDB_FUNC_TAGPRJ || + functionId == TSDB_FUNC_TS || + functionId == TSDB_FUNC_ARITHM) { + continue; + } + + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == 0) { + isAggFunc = true; + break; + } + } + + // TODO change the type, the type is not correct + if (isAggFunc) { + pCmd->type &= (~TSDB_QUERY_TYPE_PROJECTION_QUERY); + + // agg function mixed up with project query without group by exists + if (pCmd->groupbyExpr.numOfGroupCols == 0) { + return false; + } + + // get the project column + int32_t numOfPrjColumn = 0; + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); + if (pExpr->functionId == TSDB_FUNC_PRJ) { + numOfPrjColumn += 1; + + bool qualifiedCol = false; + for (int32_t j = 0; j < pCmd->groupbyExpr.numOfGroupCols; ++j) { + if (pExpr->colInfo.colId == pCmd->groupbyExpr.columnInfo[j].colId) { + qualifiedCol = true; + + pExpr->param[0].i64Key = 1; // limit the output to be 1 for each state value + pExpr->numOfParams = 1; + break; + } + } + + if (!qualifiedCol) { + setErrMsg(pCmd, msg1); + return false; + } + } + } + } + } + return true; } -void updateTagColumnIndex(SSqlCmd* pCmd) { +void updateTagColumnIndex(SSqlCmd* pCmd, int32_t tableIndex) { + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, tableIndex); + // update tags column index for group by tags - for (int32_t i = 0; i < pCmd->groupbyExpr.numOfGroupbyCols; ++i) { - int32_t index = pCmd->groupbyExpr.tagIndex[i]; + for (int32_t i = 0; i < pCmd->groupbyExpr.numOfGroupCols; ++i) { + int32_t index = pCmd->groupbyExpr.columnInfo[i].colIdx; - for (int32_t j = 0; j < pCmd->numOfReqTags; ++j) { - int32_t tagColIndex = pCmd->tagColumnIndex[j]; + for (int32_t j = 0; j < pMeterMetaInfo->numOfTags; ++j) { + int32_t tagColIndex = pMeterMetaInfo->tagColumnIndex[j]; if (tagColIndex == index) { - pCmd->groupbyExpr.tagIndex[i] = j; + pCmd->groupbyExpr.columnInfo[i].colIdx = j; break; } } @@ -2287,14 +2880,12 @@ void updateTagColumnIndex(SSqlCmd* pCmd) { // update tags column index for expression for (int32_t i = 0; i < pCmd->exprsInfo.numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); - - // not tags, continue - if (!pExpr->colInfo.isTag) { + if (!TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { // not tags, continue continue; } - for (int32_t j = 0; j < pCmd->numOfReqTags; ++j) { - if (pExpr->colInfo.colIdx == pCmd->tagColumnIndex[j]) { + for (int32_t j = 0; j < pMeterMetaInfo->numOfTags; ++j) { + if (pExpr->colInfo.colIdx == pMeterMetaInfo->tagColumnIndex[j]) { pExpr->colInfo.colIdx = j; break; } @@ -2302,146 +2893,153 @@ void updateTagColumnIndex(SSqlCmd* pCmd) { } } -int32_t setGroupByClause(SSqlCmd* pCmd, tVariantList* pList) { +int32_t parseGroupbyClause(SSqlCmd* pCmd, tVariantList* pList) { const char* msg1 = "too many columns in group by clause"; const char* msg2 = "invalid column name in group by clause"; - const char* msg3 = "functions are not available in group by query"; const char* msg4 = "group by only available for STable query"; + const char* msg5 = "group by columns must belong to one table"; + const char* msg6 = "only support group by one ordinary column"; + const char* msg7 = "not support group by expression"; + const char* msg8 = "not allowed column type for group by"; + const char* msg9 = "tags not allowed for table query"; - if (UTIL_METER_IS_NOMRAL_METER(pCmd)) { - if (pList == NULL) { - return TSDB_CODE_SUCCESS; - } else { - setErrMsg(pCmd, msg4); - return TSDB_CODE_INVALID_SQL; - } - } + // todo : handle two meter situation + SMeterMetaInfo* pMeterMetaInfo = NULL; if (pList == NULL) { return TSDB_CODE_SUCCESS; } - pCmd->groupbyExpr.numOfGroupbyCols = pList->nExpr; + pCmd->groupbyExpr.numOfGroupCols = pList->nExpr; if (pList->nExpr > TSDB_MAX_TAGS) { setErrMsg(pCmd, msg1); return TSDB_CODE_INVALID_SQL; } - SMeterMeta* pMeterMeta = pCmd->pMeterMeta; - SSchema* pSchema = tsGetSchema(pMeterMeta); + SMeterMeta* pMeterMeta = NULL; + SSchema* pSchema = NULL; + SSchema s = {0}; int32_t numOfReqTags = 0; + int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; for (int32_t i = 0; i < pList->nExpr; ++i) { tVariant* pVar = &pList->a[i].pVar; SSQLToken token = {pVar->nLen, pVar->nType, pVar->pz}; - int32_t colIdx = 0; - int16_t type = 0; - int16_t bytes = 0; - char* name = NULL; + SColumnIndex index = COLUMN_INDEX_INITIALIZER; - /* group by tbname*/ - if (strncasecmp(pVar->pz, TSQL_TBNAME_L, pVar->nLen) == 0) { - colIdx = -1; - type = TSDB_DATA_TYPE_BINARY; - bytes = TSDB_METER_NAME_LEN; - name = TSQL_TBNAME_L; - } else { - colIdx = getColumnIndexByName(&token, pSchema, pMeterMeta->numOfTags + pMeterMeta->numOfColumns); - if (colIdx < pMeterMeta->numOfColumns || colIdx > TSDB_MAX_TAGS + pMeterMeta->numOfColumns) { - setErrMsg(pCmd, msg2); - return TSDB_CODE_INVALID_SQL; - } + if (getColumnIndexByNameEx(&token, pCmd, &index) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg2); + return TSDB_CODE_INVALID_SQL; + } - type = pSchema[colIdx].type; - bytes = pSchema[colIdx].bytes; - name = pSchema[colIdx].name; - numOfReqTags++; + if (tableIndex != index.tableIndex && tableIndex >= 0) { + setErrMsg(pCmd, msg5); + return TSDB_CODE_INVALID_SQL; } - SSqlExpr* pExpr = tscSqlExprInsert(pCmd, pCmd->fieldsInfo.numOfOutputCols, TSDB_FUNC_TAG, colIdx, type, bytes); - pExpr->colInfo.isTag = true; + tableIndex = index.tableIndex; - // NOTE: tag column does not add to source column list - SColumnList ids = {0}; - insertResultField(pCmd, pCmd->fieldsInfo.numOfOutputCols, &ids, bytes, type, name); + pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); + pMeterMeta = pMeterMetaInfo->pMeterMeta; - int32_t relIndex = 0; - if (colIdx != -1) { - relIndex = colIdx - pMeterMeta->numOfColumns; - } else { // tbname - relIndex = colIdx; - } + // TODO refactor!!!!!!!!!!!!!!1 + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + s.colId = TSDB_TBNAME_COLUMN_INDEX; + s.type = TSDB_DATA_TYPE_BINARY; + s.bytes = TSDB_METER_NAME_LEN; + strcpy(s.name, TSQL_TBNAME_L); - pExpr->colInfo.colIdx = relIndex; - pCmd->groupbyExpr.tagIndex[i] = relIndex; + pSchema = &s; + } else { + pSchema = tsGetColumnSchema(pMeterMeta, index.columnIndex); + } - assert(pCmd->groupbyExpr.tagIndex[i] >= -1); - addRequiredTagColumn(pCmd, pCmd->groupbyExpr.tagIndex[i]); - } + int16_t type = 0; + int16_t bytes = 0; + char* name = NULL; - /* - * check all query functions in selection clause, multi-output functions are not available in query function - */ - for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { - int32_t functId = tscSqlExprGet(pCmd, i)->sqlFuncId; - if (IS_MULTIOUTPUT(aAggs[functId].nStatus) && functId != TSDB_FUNC_TOP_DST && functId != TSDB_FUNC_BOTTOM_DST && - functId != TSDB_FUNC_TOP && functId != TSDB_FUNC_BOTTOM) { - setErrMsg(pCmd, msg3); - return TSDB_CODE_INVALID_SQL; + bool groupTag = false; + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= pMeterMeta->numOfColumns) { + groupTag = true; } - if (functId == TSDB_FUNC_COUNT && tscSqlExprGet(pCmd, i)->colInfo.colIdx == -1) { - setErrMsg(pCmd, msg3); - return TSDB_CODE_INVALID_SQL; + if (groupTag) { + if (!UTIL_METER_IS_METRIC(pMeterMetaInfo)) { + setErrMsg(pCmd, msg9); + return TSDB_CODE_INVALID_SQL; + } + + int32_t relIndex = index.columnIndex; + if (index.columnIndex != TSDB_TBNAME_COLUMN_INDEX) { + relIndex -= pMeterMeta->numOfColumns; + } + + pCmd->groupbyExpr.columnInfo[i] = + (SColIndexEx){.colIdx = relIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId}; // relIndex; + addRequiredTagColumn(pCmd, pCmd->groupbyExpr.columnInfo[i].colIdx, index.tableIndex); + } else { + // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by + if (pSchema->type > TSDB_DATA_TYPE_BIGINT) { + setErrMsg(pCmd, msg8); + return TSDB_CODE_INVALID_SQL; + } + + tscColumnBaseInfoInsert(pCmd, &index); + pCmd->groupbyExpr.columnInfo[i] = + (SColIndexEx){.colIdx = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId}; // relIndex; + pCmd->groupbyExpr.orderType = TSQL_SO_ASC; + + if (i == 0 && pList->nExpr > 1) { + setErrMsg(pCmd, msg7); + return TSDB_CODE_INVALID_SQL; + } } } + pCmd->groupbyExpr.tableIndex = tableIndex; + return TSDB_CODE_SUCCESS; } void setColumnOffsetValueInResultset(SSqlCmd* pCmd) { - if (pCmd->metricQuery == 0) { - tscFieldInfoCalOffset(pCmd); + if (QUERY_IS_STABLE_QUERY(pCmd->type)) { + tscFieldInfoUpdateOffset(pCmd); } else { - tscFieldInfoRenewOffsetForInterResult(pCmd); + tscFieldInfoCalOffset(pCmd); } } -static int setColumnFilterInfoForTimestamp(SSqlCmd* pCmd, tVariant* pVar) { - int64_t time = 0; - const char* msg = "invalid timestamp"; +static SColumnFilterInfo* addColumnFilterInfo(SColumnBase* pColumn) { + if (pColumn == NULL) { + return NULL; + } - strdequote(pVar->pz); - char* seg = strnchr(pVar->pz, '-', pVar->nLen, false); - if (seg != NULL) { - if (taosParseTime(pVar->pz, &time, pVar->nLen, pCmd->pMeterMeta->precision) != TSDB_CODE_SUCCESS) { - setErrMsg(pCmd, msg); - return TSDB_CODE_INVALID_SQL; - } - } else { - if (tVariantDump(pVar, (char*)&time, TSDB_DATA_TYPE_BIGINT)) { - setErrMsg(pCmd, msg); - return TSDB_CODE_INVALID_SQL; - } + int32_t size = pColumn->numOfFilters + 1; + char* tmp = realloc(pColumn->filterInfo, sizeof(SColumnFilterInfo) * (size)); + if (tmp != NULL) { + pColumn->filterInfo = (SColumnFilterInfo*)tmp; } - tVariantDestroy(pVar); - tVariantCreateB(pVar, (char*)&time, 0, TSDB_DATA_TYPE_BIGINT); + pColumn->numOfFilters++; - return TSDB_CODE_SUCCESS; + SColumnFilterInfo* pColFilterInfo = &pColumn->filterInfo[pColumn->numOfFilters - 1]; + memset(pColFilterInfo, 0, sizeof(SColumnFilterInfo)); + + return pColFilterInfo; } -static int setColumnFilterInfo(SSqlCmd* pCmd, SColumnBase* pColFilter, int32_t colIdx, tSQLExpr* pExpr) { - tSQLExpr* pRight = pExpr->pRight; - SSchema* pSchema = tsGetSchema(pCmd->pMeterMeta); +static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SColumnFilterInfo* pColumnFilter, SColumnIndex* columnIndex, + tSQLExpr* pExpr) { + const char* msg = "not supported filter condition"; - pColFilter->filterOn = 1; + tSQLExpr* pRight = pExpr->pRight; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, columnIndex->tableIndex); - pColFilter->colIndex = colIdx; + SSchema* pSchema = tsGetColumnSchema(pMeterMetaInfo->pMeterMeta, columnIndex->columnIndex); - int16_t colType = pSchema[colIdx].type; + int16_t colType = pSchema->type; if (colType >= TSDB_DATA_TYPE_TINYINT && colType <= TSDB_DATA_TYPE_BIGINT) { colType = TSDB_DATA_TYPE_BIGINT; } else if (colType == TSDB_DATA_TYPE_FLOAT || colType == TSDB_DATA_TYPE_DOUBLE) { @@ -2454,83 +3052,134 @@ static int setColumnFilterInfo(SSqlCmd* pCmd, SColumnBase* pColFilter, int32_t c } if (pExpr->nSQLOptr == TK_LE || pExpr->nSQLOptr == TK_LT) { - tVariantDump(&pRight->val, (char*)&pColFilter->upperBndd, colType); - } else { // TK_GT,TK_GE,TK_EQ,TK_NE are based on the pColFilter->lowerBndd + tVariantDump(&pRight->val, (char*)&pColumnFilter->upperBndd, colType); + } else { // TK_GT,TK_GE,TK_EQ,TK_NE are based on the pColumn->lowerBndd if (colType == TSDB_DATA_TYPE_BINARY) { - pColFilter->pz = (int64_t)malloc(pRight->val.nLen + 1); - pColFilter->len = pRight->val.nLen; + pColumnFilter->pz = (int64_t)calloc(1, pRight->val.nLen + 1); + pColumnFilter->len = pRight->val.nLen; - tVariantDump(&pRight->val, (char*)pColFilter->pz, colType); - ((char*)pColFilter->pz)[pColFilter->len] = 0; + tVariantDump(&pRight->val, (char*)pColumnFilter->pz, colType); + } else if (colType == TSDB_DATA_TYPE_NCHAR) { + // pRight->val.nLen + 1 is larger than the actual nchar string length + pColumnFilter->pz = (int64_t)calloc(1, (pRight->val.nLen + 1) * TSDB_NCHAR_SIZE); + + tVariantDump(&pRight->val, (char*)pColumnFilter->pz, colType); + + size_t len = wcslen((wchar_t*)pColumnFilter->pz); + pColumnFilter->len = len * TSDB_NCHAR_SIZE; } else { - tVariantDump(&pRight->val, (char*)&pColFilter->lowerBndd, colType); + tVariantDump(&pRight->val, (char*)&pColumnFilter->lowerBndd, colType); } } switch (pExpr->nSQLOptr) { case TK_LE: - pColFilter->upperRelOptr = TSDB_RELATION_LESS_EQUAL; + pColumnFilter->upperRelOptr = TSDB_RELATION_LESS_EQUAL; break; case TK_LT: - pColFilter->upperRelOptr = TSDB_RELATION_LESS; + pColumnFilter->upperRelOptr = TSDB_RELATION_LESS; break; case TK_GT: - pColFilter->lowerRelOptr = TSDB_RELATION_LARGE; + pColumnFilter->lowerRelOptr = TSDB_RELATION_LARGE; break; case TK_GE: - pColFilter->lowerRelOptr = TSDB_RELATION_LARGE_EQUAL; + pColumnFilter->lowerRelOptr = TSDB_RELATION_LARGE_EQUAL; break; case TK_EQ: - pColFilter->lowerRelOptr = TSDB_RELATION_EQUAL; + pColumnFilter->lowerRelOptr = TSDB_RELATION_EQUAL; break; case TK_NE: - pColFilter->lowerRelOptr = TSDB_RELATION_NOT_EQUAL; + pColumnFilter->lowerRelOptr = TSDB_RELATION_NOT_EQUAL; break; case TK_LIKE: - pColFilter->lowerRelOptr = TSDB_RELATION_LIKE; + pColumnFilter->lowerRelOptr = TSDB_RELATION_LIKE; break; + default: + setErrMsg(pCmd, msg); + return TSDB_CODE_INVALID_SQL; } + return TSDB_CODE_SUCCESS; } -static int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t optr, int16_t precision); +typedef struct SCondExpr { + tSQLExpr* pTagCond; + tSQLExpr* pTimewindow; + + tSQLExpr* pColumnCond; + + tSQLExpr* pTableCond; + int16_t relType; // relation between table name in expression and other tag + // filter condition expression, TK_AND or TK_OR + int16_t tableCondIndex; + + tSQLExpr* pJoinExpr; // join condition + bool tsJoin; +} SCondExpr; -static int32_t exprToString(tSQLExpr* pExpr, char** exprString, SColumnIdList* pIdList) { +static int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t optr, int16_t timePrecision); + +static int32_t doParseWhereClause(SSqlObj* pSql, tSQLExpr** pExpr, SCondExpr* condExpr); + +static int32_t tSQLExprNodeToString(tSQLExpr* pExpr, char** str) { if (pExpr->nSQLOptr == TK_ID) { // column name - strncpy(*exprString, pExpr->colInfo.z, pExpr->colInfo.n); - *exprString += pExpr->colInfo.n; - - if (pIdList) { - bool validColumn = false; - // record and check the column name - for (int32_t i = 0; i < pIdList->numOfCols; ++i) { - int32_t len = strlen(pIdList->pSchema[i].name); - - if (pExpr->colInfo.n == len && strncasecmp(pExpr->colInfo.z, pIdList->pSchema[i].name, len) == 0) { - pIdList->ids[pIdList->numOfRecordedCols++] = (int16_t)i; - validColumn = true; - break; - } - } + strncpy(*str, pExpr->colInfo.z, pExpr->colInfo.n); + *str += pExpr->colInfo.n; - if (!validColumn) { - return TSDB_CODE_INVALID_SQL; - } - } } else if (pExpr->nSQLOptr >= TK_BOOL && pExpr->nSQLOptr <= TK_STRING) { // value - *exprString += tVariantToString(&pExpr->val, *exprString); + *str += tVariantToString(&pExpr->val, *str); + } else { + assert(false); + } + + return TSDB_CODE_SUCCESS; +} + +static bool isExprLeafNode(tSQLExpr* pExpr) { + return (pExpr->pRight == NULL && pExpr->pLeft == NULL) && + (pExpr->nSQLOptr == TK_ID || (pExpr->nSQLOptr >= TK_BOOL && pExpr->nSQLOptr <= TK_NCHAR) || + pExpr->nSQLOptr == TK_SET); +} + +static bool isExprDirectParentOfLeaftNode(tSQLExpr* pExpr) { + return (pExpr->pLeft != NULL && pExpr->pRight != NULL) && + (isExprLeafNode(pExpr->pLeft) && isExprLeafNode(pExpr->pRight)); +} + +static int32_t tSQLExprLeafToString(tSQLExpr* pExpr, bool addParentheses, char** output) { + if (!isExprDirectParentOfLeaftNode(pExpr)) { return TSDB_CODE_INVALID_SQL; } + tSQLExpr* pLeft = pExpr->pLeft; + tSQLExpr* pRight = pExpr->pRight; + + if (addParentheses) { + *(*output) = '('; + *output += 1; + } + + tSQLExprNodeToString(pLeft, output); + if (optrToString(pExpr, output) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_INVALID_SQL; + } + + tSQLExprNodeToString(pRight, output); + + if (addParentheses) { + *(*output) = ')'; + *output += 1; + } + return TSDB_CODE_SUCCESS; } static int32_t optrToString(tSQLExpr* pExpr, char** exprString) { - char le[] = "<="; - char ge[] = ">="; - char ne[] = "<>"; - char likeOptr[] = "LIKE"; + const char* le = "<="; + const char* ge = ">="; + const char* ne = "<>"; + const char* likeOptr = "LIKE"; switch (pExpr->nSQLOptr) { case TK_LE: { @@ -2587,108 +3236,245 @@ static int32_t optrToString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } -static int32_t createTableNameList(tSQLExpr* pExpr, char** queryStr) { +static int32_t tablenameListToString(tSQLExpr* pExpr, char* str) { tSQLExprList* pList = pExpr->pParam; if (pList->nExpr <= 0) { return TSDB_CODE_INVALID_SQL; } + if (pList->nExpr > 0) { + strcpy(str, QUERY_COND_REL_PREFIX_IN); + str += QUERY_COND_REL_PREFIX_IN_LEN; + } + int32_t len = 0; for (int32_t i = 0; i < pList->nExpr; ++i) { tSQLExpr* pSub = pList->a[i].pNode; - strncpy(*queryStr + len, pSub->val.pz, pSub->val.nLen); + strncpy(str + len, pSub->val.pz, pSub->val.nLen); len += pSub->val.nLen; - (*queryStr)[len++] = ','; + + if (i < pList->nExpr - 1) { + str[len++] = TBNAME_LIST_SEP[0]; + } if (pSub->val.nLen <= 0 || pSub->val.nLen > TSDB_METER_NAME_LEN) { return TSDB_CODE_INVALID_SQL; } } - *queryStr += len; - return TSDB_CODE_SUCCESS; } -static bool isTbnameToken(SSQLToken* token) { - return (strncasecmp(TSQL_TBNAME_L, token->z, token->n) == 0 && token->n == strlen(TSQL_TBNAME_L)); +static int32_t tablenameCondToString(tSQLExpr* pExpr, char* str) { + strcpy(str, QUERY_COND_REL_PREFIX_LIKE); + str += strlen(QUERY_COND_REL_PREFIX_LIKE); + + strcpy(str, pExpr->val.pz); + + return TSDB_CODE_SUCCESS; } -static int32_t buildTagQueryCondString(SSqlCmd* pCmd, tSQLExpr* pExpr, char** queryStr) { - tSQLExpr* pLeft = pExpr->pLeft; - tSQLExpr* pRight = pExpr->pRight; +enum { + TSQL_EXPR_TS = 0, + TSQL_EXPR_TAG = 1, + TSQL_EXPR_COLUMN = 2, + TSQL_EXPR_TBNAME = 3, +}; - const char* msg0 = "invalid table name list"; - const char* msg1 = "like operation is not allowed on numeric tags"; - const char* msg2 = "in and query condition cannot be mixed up"; - STagCond* pCond = &pCmd->tagCond; +static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SColumnIndex* pIndex, tSQLExpr* pExpr, int32_t sqlOptr) { + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pIndex->tableIndex); + + SMeterMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; + SSchema* pSchema = tsGetColumnSchema(pMeterMeta, pIndex->columnIndex); - if (pExpr->nSQLOptr == TK_IN && pRight->nSQLOptr == TK_SET) { - /* table name array list, invoke another routine */ - if (pCond->type == TSQL_STABLE_QTYPE_COND) { + const char* msg1 = "non binary column not support like operator"; + const char* msg2 = "binary column not support this operator"; + const char* msg3 = "OR is not supported on different column filter"; + + SColumnBase* pColumn = tscColumnBaseInfoInsert(pCmd, pIndex); + SColumnFilterInfo* pColFilter = NULL; + + /* + * in case of TK_AND filter condition, we first find the corresponding column and build the query condition together + * the already existed condition. + */ + if (sqlOptr == TK_AND) { + // this is a new filter condition on this column + if (pColumn->numOfFilters == 0) { + pColFilter = addColumnFilterInfo(pColumn); + } else { // update the existed column filter information, find the filter info here + pColFilter = &pColumn->filterInfo[0]; + } + } else if (sqlOptr == TK_OR) { + // TODO fixme: failed to invalid the filter expression: "col1 = 1 OR col2 = 2" + pColFilter = addColumnFilterInfo(pColumn); + } else { // error; + return TSDB_CODE_INVALID_SQL; + } + + pColFilter->filterOnBinary = + ((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0); + + if (pColFilter->filterOnBinary) { + if (pExpr->nSQLOptr != TK_EQ && pExpr->nSQLOptr != TK_NE && pExpr->nSQLOptr != TK_LIKE) { setErrMsg(pCmd, msg2); return TSDB_CODE_INVALID_SQL; } - - pCond->type = TSQL_STABLE_QTYPE_SET; - - if (!isTbnameToken(&pLeft->colInfo)) { + } else { + if (pExpr->nSQLOptr == TK_LIKE) { + setErrMsg(pCmd, msg1); return TSDB_CODE_INVALID_SQL; } + } + + pColumn->colIndex = *pIndex; + return doExtractColumnFilterInfo(pCmd, pColFilter, pIndex, pExpr); +} + +static void relToString(SSqlCmd* pCmd, tSQLExpr* pExpr, char** str) { + assert(pExpr->nSQLOptr == TK_AND || pExpr->nSQLOptr == TK_OR); + + const char* or = "OR"; + const char*and = "AND"; + + // if (pCmd->tagCond.relType == TSQL_STABLE_QTYPE_COND) { + if (pExpr->nSQLOptr == TK_AND) { + strcpy(*str, and); + *str += strlen(and); + } else { + strcpy(*str, or); + *str += strlen(or); + } + // } +} + +static int32_t getTagCondString(SSqlCmd* pCmd, tSQLExpr* pExpr, char** str) { + if (pExpr == NULL) { + return TSDB_CODE_SUCCESS; + } + + if (!isExprDirectParentOfLeaftNode(pExpr)) { + *(*str) = '('; + *str += 1; - int32_t ret = createTableNameList(pRight, queryStr); + int32_t ret = getTagCondString(pCmd, pExpr->pLeft, str); if (ret != TSDB_CODE_SUCCESS) { - setErrMsg(pCmd, msg0); + return ret; } + + relToString(pCmd, pExpr, str); + + ret = getTagCondString(pCmd, pExpr->pRight, str); + + *(*str) = ')'; + *str += 1; + return ret; } - // already use IN predicates - if (pCond->type == TSQL_STABLE_QTYPE_SET) { - setErrMsg(pCmd, msg2); - return TSDB_CODE_INVALID_SQL; - } else { - pCond->type = TSQL_STABLE_QTYPE_COND; + return tSQLExprLeafToString(pExpr, true, str); +} + +static int32_t getTablenameCond(SSqlCmd* pCmd, tSQLExpr* pTableCond, char* str) { + const char* msg0 = "invalid table name list"; + + if (pTableCond == NULL) { + return TSDB_CODE_SUCCESS; } - *(*queryStr) = '('; - *queryStr += 1; + tSQLExpr* pLeft = pTableCond->pLeft; + tSQLExpr* pRight = pTableCond->pRight; - exprToString(pLeft, queryStr, NULL); - if (optrToString(pExpr, queryStr) != TSDB_CODE_SUCCESS) { + if (!isTablenameToken(&pLeft->colInfo)) { return TSDB_CODE_INVALID_SQL; } - /* pattern string is too long */ - if (pExpr->nSQLOptr == TK_LIKE) { - if (pRight->val.nLen > TSDB_PATTERN_STRING_MAX_LEN) { - return TSDB_CODE_INVALID_SQL; - } + int32_t ret = TSDB_CODE_SUCCESS; - SSchema* pSchema = tsGetSchema(pCmd->pMeterMeta); + if (pTableCond->nSQLOptr == TK_IN) { + ret = tablenameListToString(pRight, str); + } else if (pTableCond->nSQLOptr == TK_LIKE) { + ret = tablenameCondToString(pRight, str); + } - int32_t numOfCols = pCmd->pMeterMeta->numOfColumns; - int32_t numOfTags = pCmd->pMeterMeta->numOfTags; + if (ret != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg0); + } - int32_t colIdx = getColumnIndexByName(&pLeft->colInfo, pSchema, numOfCols + numOfTags); - if ((!isTbnameToken(&pLeft->colInfo)) && pSchema[colIdx].type != TSDB_DATA_TYPE_BINARY && - pSchema[colIdx].type != TSDB_DATA_TYPE_NCHAR) { - setErrMsg(pCmd, msg1); + return ret; +} + +static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, tSQLExpr* pExpr, int32_t relOptr) { + if (pExpr == NULL) { + return TSDB_CODE_SUCCESS; + } + + if (!isExprDirectParentOfLeaftNode(pExpr)) { // internal node + int32_t ret = getColumnQueryCondInfo(pCmd, pExpr->pLeft, pExpr->nSQLOptr); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + return getColumnQueryCondInfo(pCmd, pExpr->pRight, pExpr->nSQLOptr); + } else { // handle leaf node + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByNameEx(&pExpr->pLeft->colInfo, pCmd, &index) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } + + return extractColumnFilterInfo(pCmd, &index, pExpr, relOptr); } +} - exprToString(pRight, queryStr, NULL); +static int32_t getJoinCondInfo(SSqlObj* pSql, tSQLExpr* pExpr) { + const char* msg = "invalid join query condition"; - *(*queryStr) = ')'; - *queryStr += 1; + if (pExpr == NULL) { + return TSDB_CODE_SUCCESS; + } + SSqlCmd* pCmd = &pSql->cmd; + + if (!isExprDirectParentOfLeaftNode(pExpr)) { + setErrMsg(pCmd, msg); + return TSDB_CODE_INVALID_SQL; + } + + STagCond* pTagCond = &pCmd->tagCond; + SJoinNode* pLeft = &pTagCond->joinInfo.left; + SJoinNode* pRight = &pTagCond->joinInfo.right; + + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByNameEx(&pExpr->pLeft->colInfo, pCmd, &index) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_INVALID_SQL; + } + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); + int16_t tagColIndex = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; + + pLeft->uid = pMeterMetaInfo->pMeterMeta->uid; + pLeft->tagCol = tagColIndex; + strcpy(pLeft->meterId, pMeterMetaInfo->name); + + index = (SColumnIndex)COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByNameEx(&pExpr->pRight->colInfo, pCmd, &index) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_INVALID_SQL; + } + + pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); + tagColIndex = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; + + pRight->uid = pMeterMetaInfo->pMeterMeta->uid; + pRight->tagCol = tagColIndex; + strcpy(pRight->meterId, pMeterMetaInfo->name); + + pTagCond->joinInfo.hasJoin = true; return TSDB_CODE_SUCCESS; } // todo error handle / such as and /or mixed with +/-/*/ -int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString, SColumnIdList* colIdList) { +int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString) { tSQLExpr* pLeft = pExpr->pLeft; tSQLExpr* pRight = pExpr->pRight; @@ -2696,9 +3482,9 @@ int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString, SColumnIdL *exprString += 1; if (pLeft->nSQLOptr >= TK_PLUS && pLeft->nSQLOptr <= TK_REM) { - buildArithmeticExprString(pLeft, exprString, colIdList); + buildArithmeticExprString(pLeft, exprString); } else { - int32_t ret = exprToString(pLeft, exprString, colIdList); + int32_t ret = tSQLExprNodeToString(pLeft, exprString); if (ret != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } @@ -2707,9 +3493,9 @@ int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString, SColumnIdL optrToString(pExpr, exprString); if (pRight->nSQLOptr >= TK_PLUS && pRight->nSQLOptr <= TK_REM) { - buildArithmeticExprString(pRight, exprString, colIdList); + buildArithmeticExprString(pRight, exprString); } else { - int32_t ret = exprToString(pRight, exprString, colIdList); + int32_t ret = tSQLExprNodeToString(pRight, exprString); if (ret != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } @@ -2721,15 +3507,23 @@ int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString, SColumnIdL return TSDB_CODE_SUCCESS; } -static int32_t validateSQLExpr(tSQLExpr* pExpr, SSchema* pSchema, int32_t numOfCols) { +static int32_t validateSQLExpr(tSQLExpr* pExpr, SSchema* pSchema, int32_t numOfCols, SColumnIdListRes* pList) { if (pExpr->nSQLOptr == TK_ID) { bool validColumnName = false; + + SColumnList* list = &pList->list; + for (int32_t i = 0; i < numOfCols; ++i) { if (strncasecmp(pExpr->colInfo.z, pSchema[i].name, pExpr->colInfo.n) == 0 && pExpr->colInfo.n == strlen(pSchema[i].name)) { if (pSchema[i].type < TSDB_DATA_TYPE_TINYINT || pSchema[i].type > TSDB_DATA_TYPE_DOUBLE) { return TSDB_CODE_INVALID_SQL; } + + if (pList != NULL) { + list->ids[list->num++].columnIndex = (int16_t)i; + } + validColumnName = true; } } @@ -2740,24 +3534,27 @@ static int32_t validateSQLExpr(tSQLExpr* pExpr, SSchema* pSchema, int32_t numOfC } else if (pExpr->nSQLOptr == TK_FLOAT && (isnan(pExpr->val.dKey) || isinf(pExpr->val.dKey))) { return TSDB_CODE_INVALID_SQL; + } else if (pExpr->nSQLOptr >= TK_MIN && pExpr->nSQLOptr <= TK_LAST_ROW) { + return TSDB_CODE_INVALID_SQL; } return TSDB_CODE_SUCCESS; } -static int32_t validateArithmeticSQLExpr(tSQLExpr* pExpr, SSchema* pSchema, int32_t numOfCols) { +static int32_t validateArithmeticSQLExpr(tSQLExpr* pExpr, SSchema* pSchema, int32_t numOfCols, + SColumnIdListRes* pList) { if (pExpr == NULL) { return TSDB_CODE_SUCCESS; } tSQLExpr* pLeft = pExpr->pLeft; if (pLeft->nSQLOptr >= TK_PLUS && pLeft->nSQLOptr <= TK_REM) { - int32_t ret = validateArithmeticSQLExpr(pLeft, pSchema, numOfCols); + int32_t ret = validateArithmeticSQLExpr(pLeft, pSchema, numOfCols, pList); if (ret != TSDB_CODE_SUCCESS) { return ret; } } else { - int32_t ret = validateSQLExpr(pLeft, pSchema, numOfCols); + int32_t ret = validateSQLExpr(pLeft, pSchema, numOfCols, pList); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -2765,12 +3562,12 @@ static int32_t validateArithmeticSQLExpr(tSQLExpr* pExpr, SSchema* pSchema, int3 tSQLExpr* pRight = pExpr->pRight; if (pRight->nSQLOptr >= TK_PLUS && pRight->nSQLOptr <= TK_REM) { - int32_t ret = validateArithmeticSQLExpr(pRight, pSchema, numOfCols); + int32_t ret = validateArithmeticSQLExpr(pRight, pSchema, numOfCols, pList); if (ret != TSDB_CODE_SUCCESS) { return ret; } } else { - int32_t ret = validateSQLExpr(pRight, pSchema, numOfCols); + int32_t ret = validateSQLExpr(pRight, pSchema, numOfCols, pList); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -2786,16 +3583,14 @@ static bool isValidExpr(tSQLExpr* pLeft, tSQLExpr* pRight, int32_t optr) { /* * filter illegal expression in where clause: - * 1. columnA = columnB - * 2. count(*) > 12 - * 3. sum(columnA) > sum(columnB) - * 4. 4 < 5, 'ABC'>'abc' + * 1. count(*) > 12 + * 2. sum(columnA) > sum(columnB) + * 3. 4 < 5, 'ABC'>'abc' * * However, columnA < 4+12 is valid */ - if ((pLeft->nSQLOptr == TK_ID && pRight->nSQLOptr == TK_ID) || - (pLeft->nSQLOptr >= TK_COUNT && pLeft->nSQLOptr <= TK_WAVG) || - (pRight->nSQLOptr >= TK_COUNT && pRight->nSQLOptr <= TK_WAVG) || + if ((pLeft->nSQLOptr >= TK_COUNT && pLeft->nSQLOptr <= TK_LAST_ROW) || + (pRight->nSQLOptr >= TK_COUNT && pRight->nSQLOptr <= TK_LAST_ROW) || (pLeft->nSQLOptr >= TK_BOOL && pLeft->nSQLOptr <= TK_BINARY && pRight->nSQLOptr >= TK_BOOL && pRight->nSQLOptr <= TK_BINARY)) { return false; @@ -2804,235 +3599,370 @@ static bool isValidExpr(tSQLExpr* pLeft, tSQLExpr* pRight, int32_t optr) { return true; } -static int32_t getColumnFilterInfo(SSqlCmd* pCmd, int32_t colIdx, tSQLExpr* pExpr) { - SMeterMeta* pMeterMeta = pCmd->pMeterMeta; - SSchema* pSchema = tsGetSchema(pMeterMeta); +static void exchangeExpr(tSQLExpr* pExpr) { + tSQLExpr* pLeft = pExpr->pLeft; + tSQLExpr* pRight = pExpr->pRight; - const char* msg = "nchar column not available for filter"; - const char* msg1 = "non binary column not support like operator"; - const char* msg2 = "binary column not support this operator"; - const char* msg3 = "column not support in operator"; + if (pRight->nSQLOptr == TK_ID && (pLeft->nSQLOptr == TK_INTEGER || pLeft->nSQLOptr == TK_FLOAT || + pLeft->nSQLOptr == TK_STRING || pLeft->nSQLOptr == TK_BOOL)) { + /* + * exchange value of the left handside and the value of the right-handside + * to make sure that the value of filter expression always locates in + * right-handside and + * the column-id is at the left handside. + */ + uint32_t optr = 0; + switch (pExpr->nSQLOptr) { + case TK_LE: + optr = TK_GE; + break; + case TK_LT: + optr = TK_GT; + break; + case TK_GT: + optr = TK_LT; + break; + case TK_GE: + optr = TK_LE; + break; + default: + optr = pExpr->nSQLOptr; + } - if (pSchema[colIdx].type == TSDB_DATA_TYPE_NCHAR) { - setErrMsg(pCmd, msg); - return -1; + pExpr->nSQLOptr = optr; + SWAP(pExpr->pLeft, pExpr->pRight, void*); + } +} + +static bool validateJoinExprNode(SSqlCmd* pCmd, tSQLExpr* pExpr, SColumnIndex* pLeftIndex) { + const char* msg1 = "illegal column name"; + const char* msg2 = "= is expected in join expression"; + const char* msg3 = "join column must have same type"; + const char* msg4 = "self join is not allowed"; + const char* msg5 = "join table must be the same type(table to table, super table to super table)"; + const char* msg6 = "tags in join condition not support binary/nchar types"; + + tSQLExpr* pRight = pExpr->pRight; + + if (pRight->nSQLOptr != TK_ID) { + return true; + } + + if (pExpr->nSQLOptr != TK_EQ) { + setErrMsg(pCmd, msg2); + return false; + } + + SColumnIndex rightIndex = COLUMN_INDEX_INITIALIZER; + + if (getColumnIndexByNameEx(&pRight->colInfo, pCmd, &rightIndex) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg1); + return false; } - if (pExpr->nSQLOptr == TK_IN) { + // todo extract function + SMeterMetaInfo* pLeftMeterMeta = tscGetMeterMetaInfo(pCmd, pLeftIndex->tableIndex); + SSchema* pLeftSchema = tsGetSchema(pLeftMeterMeta->pMeterMeta); + int16_t leftType = pLeftSchema[pLeftIndex->columnIndex].type; + + SMeterMetaInfo* pRightMeterMeta = tscGetMeterMetaInfo(pCmd, rightIndex.tableIndex); + SSchema* pRightSchema = tsGetSchema(pRightMeterMeta->pMeterMeta); + int16_t rightType = pRightSchema[rightIndex.columnIndex].type; + + if (leftType != rightType) { setErrMsg(pCmd, msg3); - return -1; + return false; + } else if (pLeftIndex->tableIndex == rightIndex.tableIndex) { + setErrMsg(pCmd, msg4); + return false; + } else if (leftType == TSDB_DATA_TYPE_BINARY || leftType == TSDB_DATA_TYPE_NCHAR) { + setErrMsg(pCmd, msg6); + return false; } - SColumnBase* pColFilter = tscColumnInfoInsert(pCmd, colIdx); + // table to table/ super table to super table are allowed + if (UTIL_METER_IS_METRIC(pLeftMeterMeta) != UTIL_METER_IS_METRIC(pRightMeterMeta)) { + setErrMsg(pCmd, msg5); + return false; + } - pColFilter->filterOnBinary = ((pSchema[colIdx].type == TSDB_DATA_TYPE_BINARY) ? 1 : 0); + return true; +} - if (pColFilter->filterOnBinary) { - if (pExpr->nSQLOptr != TK_EQ && pExpr->nSQLOptr != TK_NE && pExpr->nSQLOptr != TK_LIKE) { - setErrMsg(pCmd, msg2); - return TSDB_CODE_INVALID_SQL; - } - } else { - if (pExpr->nSQLOptr == TK_LIKE) { - setErrMsg(pCmd, msg1); - return TSDB_CODE_INVALID_SQL; +static bool validTableNameOptr(tSQLExpr* pExpr) { + const char nameFilterOptr[] = {TK_IN, TK_LIKE}; + + for (int32_t i = 0; i < tListLen(nameFilterOptr); ++i) { + if (pExpr->nSQLOptr == nameFilterOptr[i]) { + return true; } } - return setColumnFilterInfo(pCmd, pColFilter, colIdx, pExpr); + return false; } -static int32_t handleExprInQueryCond(SSqlCmd* pCmd, bool* queryTimeRangeIsSet, char** queryStr, int64_t* stime, - int64_t* etime, tSQLExpr* pExpr) { - tSQLExpr* pLeft = pExpr->pLeft; - tSQLExpr* pRight = pExpr->pRight; +static int32_t setExprToCond(SSqlCmd* pCmd, tSQLExpr** parent, tSQLExpr* pExpr, const char* msg, int32_t parentOptr) { + if (*parent != NULL) { + if (parentOptr == TK_OR && msg != NULL) { + setErrMsg(pCmd, msg); + return TSDB_CODE_INVALID_SQL; + } - SMeterMeta* pMeterMeta = pCmd->pMeterMeta; - SSchema* pSchema = tsGetSchema(pMeterMeta); + *parent = tSQLExprCreate((*parent), pExpr, parentOptr); + } else { + *parent = pExpr; + } - int32_t numOfCols = pMeterMeta->numOfColumns; - int32_t numOfTags = pMeterMeta->numOfTags; + return TSDB_CODE_SUCCESS; +} - const char* msg = "meter query cannot use tags filter"; - const char* msg1 = "illegal column name"; - const char* msg2 = "invalid timestamp"; +static int32_t handleExprInQueryCond(SSqlCmd* pCmd, tSQLExpr** pExpr, SCondExpr* pCondExpr, int32_t* type, + int32_t parentOptr) { + const char* msg1 = "meter query cannot use tags filter"; + const char* msg2 = "illegal column name"; + const char* msg3 = "only one query time range allowed"; + const char* msg4 = "only one join condition allowed"; + const char* msg5 = "AND is allowed to filter on different ordinary columns"; + const char* msg6 = "not support ordinary column join"; + const char* msg7 = "only one query condition on tbname allowed"; + const char* msg8 = "only in/like allowed in filter table name"; - int32_t colIdx = getColumnIndexByName(&pLeft->colInfo, pSchema, numOfCols + numOfTags); - bool istbname = isTbnameToken(&pLeft->colInfo); + tSQLExpr* pLeft = (*pExpr)->pLeft; + tSQLExpr* pRight = (*pExpr)->pRight; - if (colIdx < 0 && (!istbname)) { - setErrMsg(pCmd, msg1); + int32_t ret = TSDB_CODE_SUCCESS; + + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByNameEx(&pLeft->colInfo, pCmd, &index) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg2); return TSDB_CODE_INVALID_SQL; } - if (colIdx == 0) { // query on time range - *queryTimeRangeIsSet = true; - if (getTimeRange(stime, etime, pRight, pExpr->nSQLOptr, pMeterMeta->precision) != TSDB_CODE_SUCCESS) { - setErrMsg(pCmd, msg2); + assert(isExprDirectParentOfLeaftNode(*pExpr)); + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); + SMeterMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; + + if (index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // query on time range + if (!validateJoinExprNode(pCmd, *pExpr, &index)) { return TSDB_CODE_INVALID_SQL; } - } else if (colIdx >= numOfCols || istbname) { // query on tags - if (UTIL_METER_IS_NOMRAL_METER(pCmd)) { - setErrMsg(pCmd, msg); + + // set join query condition + if (pRight->nSQLOptr == TK_ID) { // no need to keep the timestamp join condition + pCmd->type |= TSDB_QUERY_TYPE_JOIN_QUERY; + pCondExpr->tsJoin = true; + + /* + * to release expression, e.g., m1.ts = m2.ts, + * since this expression is used to set the join query type + */ + tSQLExprDestroy(*pExpr); + } else { + ret = setExprToCond(pCmd, &pCondExpr->pTimewindow, *pExpr, msg3, parentOptr); + } + + *pExpr = NULL; // remove this expression + *type = TSQL_EXPR_TS; + } else if (index.columnIndex >= pMeterMeta->numOfColumns || + index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { // query on tags + // check for tag query condition + if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { + setErrMsg(pCmd, msg1); + return TSDB_CODE_INVALID_SQL; + } + + // check for like expression + if ((*pExpr)->nSQLOptr == TK_LIKE) { + if (pRight->val.nLen > TSDB_PATTERN_STRING_MAX_LEN) { + return TSDB_CODE_INVALID_SQL; + } + + SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + + if ((!isTablenameToken(&pLeft->colInfo)) && pSchema[index.columnIndex].type != TSDB_DATA_TYPE_BINARY && + pSchema[index.columnIndex].type != TSDB_DATA_TYPE_NCHAR) { + setErrMsg(pCmd, msg2); + return TSDB_CODE_INVALID_SQL; + } + } + + // in case of in operator, keep it in a seperate attribute + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (!validTableNameOptr(*pExpr)) { + setErrMsg(pCmd, msg8); + return TSDB_CODE_INVALID_SQL; + } + + if (pCondExpr->pTableCond == NULL) { + pCondExpr->pTableCond = *pExpr; + pCondExpr->relType = parentOptr; + pCondExpr->tableCondIndex = index.tableIndex; + } else { + setErrMsg(pCmd, msg7); + return TSDB_CODE_INVALID_SQL; + } + + *type = TSQL_EXPR_TBNAME; + *pExpr = NULL; + } else { + if (pRight->nSQLOptr == TK_ID) { // join on tag columns for stable query + if (!validateJoinExprNode(pCmd, *pExpr, &index)) { + return TSDB_CODE_INVALID_SQL; + } + + if (pCondExpr->pJoinExpr != NULL) { + setErrMsg(pCmd, msg4); + return TSDB_CODE_INVALID_SQL; + } + + pCmd->type |= TSDB_QUERY_TYPE_JOIN_QUERY; + ret = setExprToCond(pCmd, &pCondExpr->pJoinExpr, *pExpr, NULL, parentOptr); + *pExpr = NULL; + } else { + // do nothing + // ret = setExprToCond(pCmd, &pCondExpr->pTagCond, + // *pExpr, NULL, parentOptr); + } + + *type = TSQL_EXPR_TAG; + } + + } else { // query on other columns + *type = TSQL_EXPR_COLUMN; + + if (pRight->nSQLOptr == TK_ID) { // other column cannot be served as the join column + setErrMsg(pCmd, msg6); return TSDB_CODE_INVALID_SQL; } - return buildTagQueryCondString(pCmd, pExpr, queryStr); - } else { // query on other columns - return getColumnFilterInfo(pCmd, colIdx, pExpr); - } - return TSDB_CODE_SUCCESS; -} + // if (parentOptr == TK_OR) { + // setErrMsg(pCmd, msg5); + // return TSDB_CODE_INVALID_SQL; + // } -static void insertLeftParentheses(char** queryStr, char* p) { - int32_t len = (*queryStr - p); - memmove(p + 1, p, len); - p[0] = '('; - *queryStr += 1; -} + ret = setExprToCond(pCmd, &pCondExpr->pColumnCond, *pExpr, NULL, parentOptr); + *pExpr = NULL; // remove it from expr tree + } -static void removeLeftParentheses(char** queryStr, char* p) { - // remove the left parentheses - memmove(p, p + 1, *queryStr - p - 1); - *queryStr -= 1; + return ret; } -int32_t getQueryCondExprImpl(SSqlCmd* pCmd, tSQLExpr* pExpr, int64_t* stime, int64_t* etime, bool* queryTimeRangeIsSet, - char** queryStr) { +int32_t getQueryCondExpr(SSqlCmd* pCmd, tSQLExpr** pExpr, SCondExpr* pCondExpr, int32_t* type, int32_t parentOptr) { if (pExpr == NULL) { return TSDB_CODE_SUCCESS; } - tSQLExpr* pLeft = pExpr->pLeft; - tSQLExpr* pRight = pExpr->pRight; + const char* msg1 = "query condition between different columns must use 'AND'"; - if (!isValidExpr(pLeft, pRight, pExpr->nSQLOptr)) { + tSQLExpr* pLeft = (*pExpr)->pLeft; + tSQLExpr* pRight = (*pExpr)->pRight; + + if (!isValidExpr(pLeft, pRight, (*pExpr)->nSQLOptr)) { return TSDB_CODE_INVALID_SQL; } - if (pExpr->nSQLOptr == TK_AND || pExpr->nSQLOptr == TK_OR) { - int64_t stime1 = 0, etime1 = INT64_MAX; - bool tmRangeIsSet = false; + int32_t leftType = -1; + int32_t rightType = -1; - char* p = *queryStr; - int32_t ret = getQueryCondExprImpl(pCmd, pExpr->pLeft, &stime1, &etime1, &tmRangeIsSet, queryStr); + if (!isExprDirectParentOfLeaftNode(*pExpr)) { + int32_t ret = getQueryCondExpr(pCmd, &(*pExpr)->pLeft, pCondExpr, &leftType, (*pExpr)->nSQLOptr); if (ret != TSDB_CODE_SUCCESS) { return ret; } - if (tmRangeIsSet) { - *stime = stime1; - *etime = etime1; + ret = getQueryCondExpr(pCmd, &(*pExpr)->pRight, pCondExpr, &rightType, (*pExpr)->nSQLOptr); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } - *queryTimeRangeIsSet = true; + /* + * if left child and right child do not belong to the same group, the sub + * expression is not valid for parent node, it must be TK_AND operator. + */ + if (leftType != rightType) { + if ((*pExpr)->nSQLOptr == TK_OR && (leftType + rightType != TSQL_EXPR_TBNAME + TSQL_EXPR_TAG)) { + setErrMsg(pCmd, msg1); + return TSDB_CODE_INVALID_SQL; + } } - if (p == *queryStr) { - /* - * query on timestamp or filter on normal columns - * no data serialize to string - * - * do nothing - */ - } else { // serialize relational operator for tag filter operation - if (pCmd->tagCond.type == TSQL_STABLE_QTYPE_SET) { - /* using id in clause, and/or is not needed */ + *type = rightType; + return TSDB_CODE_SUCCESS; + } - } else { - assert(pCmd->tagCond.type == TSQL_STABLE_QTYPE_COND); - insertLeftParentheses(queryStr, p); + exchangeExpr(*pExpr); - char* optr = (pExpr->nSQLOptr == TK_AND) ? "and" : "or"; - int32_t len = (pExpr->nSQLOptr == TK_AND) ? 3 : 2; - strcpy(*queryStr, optr); + return handleExprInQueryCond(pCmd, pExpr, pCondExpr, type, parentOptr); +} - *queryStr += len; - } - } +static void doCompactQueryExpr(tSQLExpr** pExpr) { + if (*pExpr == NULL || isExprDirectParentOfLeaftNode(*pExpr)) { + return; + } - int64_t stime2 = 0, etime2 = INT64_MAX; - tmRangeIsSet = false; - char* p2 = *queryStr; - ret = getQueryCondExprImpl(pCmd, pExpr->pRight, &stime2, &etime2, &tmRangeIsSet, queryStr); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } + if ((*pExpr)->pLeft) { + doCompactQueryExpr(&(*pExpr)->pLeft); + } - if (tmRangeIsSet) { - *queryTimeRangeIsSet = true; - if (pExpr->nSQLOptr == TK_AND) { - *stime = stime2 > (*stime) ? stime2 : (*stime); - *etime = etime2 < (*etime) ? etime2 : (*etime); + if ((*pExpr)->pRight) { + doCompactQueryExpr(&(*pExpr)->pRight); + } - } else { - const char* msg1 = "not support multi-segments query time ranges"; - setErrMsg(pCmd, msg1); - return TSDB_CODE_INVALID_SQL; - } - } + if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight == NULL && + ((*pExpr)->nSQLOptr == TK_OR || (*pExpr)->nSQLOptr == TK_AND)) { + tSQLExprNodeDestroy(*pExpr); + *pExpr = NULL; - if (p != *queryStr) { // either the left and right hand side has tags - // filter - if (p2 == *queryStr && p != p2 && pCmd->tagCond.type == TSQL_STABLE_QTYPE_COND) { - /* - * has no tags filter info on the right hand side - * has filter on the left hand side - * - * rollback string - */ - int32_t len = (pExpr->nSQLOptr == TK_AND) ? 3 : 2; - *queryStr -= len; + } else if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight != NULL) { + tSQLExpr* tmpPtr = (*pExpr)->pRight; + tSQLExprNodeDestroy(*pExpr); - removeLeftParentheses(queryStr, p); - } else if (p2 != *queryStr && p == p2) { - // do nothing - } else { - if (pCmd->tagCond.type == TSQL_STABLE_QTYPE_COND) { - *(*queryStr) = ')'; - *queryStr += 1; - } - } - } + (*pExpr) = tmpPtr; + } else if ((*pExpr)->pRight == NULL && (*pExpr)->pLeft != NULL) { + tSQLExpr* tmpPtr = (*pExpr)->pLeft; + tSQLExprNodeDestroy(*pExpr); - return TSDB_CODE_SUCCESS; + (*pExpr) = tmpPtr; } +} - if (pLeft->nSQLOptr == TK_ID && (pRight->nSQLOptr == TK_INTEGER || pRight->nSQLOptr == TK_FLOAT || - pRight->nSQLOptr == TK_STRING || pRight->nSQLOptr == TK_BOOL)) { - // do nothing - } else if (pRight->nSQLOptr == TK_ID && (pLeft->nSQLOptr == TK_INTEGER || pLeft->nSQLOptr == TK_FLOAT || - pLeft->nSQLOptr == TK_STRING || pLeft->nSQLOptr == TK_BOOL)) { - /* - * exchange value of the left-handside and the value of the right-handside - * to make sure that the value of filter expression always locates in right-handside and - * the column-id is at the left hande side. - */ - uint32_t optr = 0; - switch (pExpr->nSQLOptr) { - case TK_LE: - optr = TK_GE; - break; - case TK_LT: - optr = TK_GT; - break; - case TK_GT: - optr = TK_LT; - break; - case TK_GE: - optr = TK_LE; - break; - default: - optr = pExpr->nSQLOptr; +static void doExtractExprForSTable(tSQLExpr** pExpr, SSqlCmd* pCmd, tSQLExpr** pOut, int32_t tableIndex) { + if (isExprDirectParentOfLeaftNode(*pExpr)) { + tSQLExpr* pLeft = (*pExpr)->pLeft; + + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByNameEx(&pLeft->colInfo, pCmd, &index) != TSDB_CODE_SUCCESS) { + return; } - pExpr->nSQLOptr = optr; + if (index.tableIndex != tableIndex) { + return; + } + + SSQLToken t = {0}; + extractTableNameFromToken(&pLeft->colInfo, &t); + + *pOut = *pExpr; + (*pExpr) = NULL; + + } else { + *pOut = tSQLExprCreate(NULL, NULL, (*pExpr)->nSQLOptr); + + doExtractExprForSTable(&(*pExpr)->pLeft, pCmd, &((*pOut)->pLeft), tableIndex); + doExtractExprForSTable(&(*pExpr)->pRight, pCmd, &((*pOut)->pRight), tableIndex); + } +} + +static tSQLExpr* extractExprForSTable(tSQLExpr** pExpr, SSqlCmd* pCmd, int32_t tableIndex) { + tSQLExpr* pResExpr = NULL; - tSQLExpr* pTmpExpr = pExpr->pLeft; - pExpr->pLeft = pExpr->pRight; - pExpr->pRight = pTmpExpr; + if (*pExpr != NULL) { + doExtractExprForSTable(pExpr, pCmd, &pResExpr, tableIndex); + doCompactQueryExpr(&pResExpr); } - return handleExprInQueryCond(pCmd, queryTimeRangeIsSet, queryStr, stime, etime, pExpr); + return pResExpr; } int tableNameCompar(const void* lhs, const void* rhs) { @@ -3048,19 +3978,40 @@ int tableNameCompar(const void* lhs, const void* rhs) { return ret > 0 ? 1 : -1; } -static int32_t setMetersIDForMetricQuery(SSqlObj* pSql, char* tmpTagCondBuf) { +static int32_t setTableCondForMetricQuery(SSqlObj* pSql, tSQLExpr* pExpr, int16_t tableCondIndex, + char* tmpTableCondBuf) { SSqlCmd* pCmd = &pSql->cmd; const char* msg = "meter name too long"; - pCmd->tagCond.allocSize = 4096; - pCmd->tagCond.pData = realloc(pCmd->tagCond.pData, pCmd->tagCond.allocSize); + if (pExpr == NULL) { + return TSDB_CODE_SUCCESS; + } + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, tableCondIndex); + + STagCond* pTagCond = &pSql->cmd.tagCond; + pTagCond->tbnameCond.uid = pMeterMetaInfo->pMeterMeta->uid; + + SString* pTableCond = &pCmd->tagCond.tbnameCond.cond; + SStringAlloc(pTableCond, 4096); + + assert(pExpr->nSQLOptr == TK_LIKE || pExpr->nSQLOptr == TK_IN); + + if (pExpr->nSQLOptr == TK_LIKE) { + strcpy(pTableCond->z, tmpTableCondBuf); + pTableCond->n = strlen(pTableCond->z); + return TSDB_CODE_SUCCESS; + } + + strcpy(pTableCond->z, QUERY_COND_REL_PREFIX_IN); + pTableCond->n += strlen(QUERY_COND_REL_PREFIX_IN); char db[TSDB_METER_ID_LEN] = {0}; - /* remove the duplicated input table names */ + // remove the duplicated input table names int32_t num = 0; - char** segments = strsplit(tmpTagCondBuf, ",", &num); - qsort(segments, num, POINTER_BYTES, tableNameCompar); + char** segments = strsplit(tmpTableCondBuf + QUERY_COND_REL_PREFIX_IN_LEN, TBNAME_LIST_SEP, &num); + qsort(segments, num, sizeof(void*), tableNameCompar); int32_t j = 1; for (int32_t i = 1; i < num; ++i) { @@ -3070,37 +4021,27 @@ static int32_t setMetersIDForMetricQuery(SSqlObj* pSql, char* tmpTagCondBuf) { } num = j; - extractDBName(pCmd->name, db); - SSQLToken tDB = { - .z = db, - .n = strlen(db), - .type = TK_STRING, - }; + SSQLToken dbToken = extractDBName(pMeterMetaInfo->name, db); + char* acc = getAccountId(pSql); - char* acc = getAccountId(pSql); for (int32_t i = 0; i < num; ++i) { - if (pCmd->tagCond.allocSize - pCmd->tagCond.len < (TSDB_METER_ID_LEN + 1)) { - /* remain space is insufficient, buy more spaces */ - pCmd->tagCond.allocSize = (pCmd->tagCond.allocSize << 1); - pCmd->tagCond.pData = realloc(pCmd->tagCond.pData, pCmd->tagCond.allocSize); - } + SStringEnsureRemain(pTableCond, TSDB_METER_ID_LEN); if (i >= 1) { - pCmd->tagCond.pData[pCmd->tagCond.len++] = ','; + pTableCond->z[pTableCond->n++] = TBNAME_LIST_SEP[0]; } - int32_t xlen = strlen(segments[i]); - + int32_t xlen = strlen(segments[i]); SSQLToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; - int32_t ret = setObjFullName(pCmd->tagCond.pData + pCmd->tagCond.len, acc, &tDB, &t, &xlen); + int32_t ret = setObjFullName(pTableCond->z + pTableCond->n, acc, &dbToken, &t, &xlen); if (ret != TSDB_CODE_SUCCESS) { - setErrMsg(pCmd, msg); tfree(segments); + setErrMsg(pCmd, msg); return ret; } - pCmd->tagCond.len += xlen; + pTableCond->n += xlen; } tfree(segments); @@ -3111,9 +4052,10 @@ static bool validateFilterExpr(SSqlCmd* pCmd) { for (int32_t i = 0; i < pCmd->colList.numOfCols; ++i) { SColumnBase* pColBase = &pCmd->colList.pColList[i]; - if (pColBase->filterOn > 0) { - int32_t lowerOptr = pColBase->lowerRelOptr; - int32_t upperOptr = pColBase->upperRelOptr; + for (int32_t j = 0; j < pColBase->numOfFilters; ++j) { + SColumnFilterInfo* pColFilter = &pColBase->filterInfo[j]; + int32_t lowerOptr = pColFilter->lowerRelOptr; + int32_t upperOptr = pColFilter->upperRelOptr; if ((lowerOptr == TSDB_RELATION_LARGE_EQUAL || lowerOptr == TSDB_RELATION_LARGE) && (upperOptr == TSDB_RELATION_LESS_EQUAL || upperOptr == TSDB_RELATION_LESS)) { @@ -3130,66 +4072,228 @@ static bool validateFilterExpr(SSqlCmd* pCmd) { return true; } -int32_t buildQueryCond(SSqlObj* pSql, tSQLExpr* pExpr) { - SSqlCmd* pCmd = &pSql->cmd; +static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, tSQLExpr* pExpr) { + const char* msg0 = "invalid timestamp"; + const char* msg1 = "only one time stamp window allowed"; if (pExpr == NULL) { return TSDB_CODE_SUCCESS; } - const char* msg1 = "invalid expression"; - const char* msg2 = "meter is not allowed"; - const char* msg3 = "invalid filter expression"; + if (!isExprDirectParentOfLeaftNode(pExpr)) { + if (pExpr->nSQLOptr == TK_OR) { + setErrMsg(pCmd, msg1); + return TSDB_CODE_INVALID_SQL; + } - tSQLExpr* pLeft = pExpr->pLeft; - tSQLExpr* pRight = pExpr->pRight; - if (pLeft == NULL || pRight == NULL || (pLeft->nSQLOptr == TK_ID && pRight->nSQLOptr == TK_ID)) { - setErrMsg(pCmd, msg1); + getTimeRangeFromExpr(pCmd, pExpr->pLeft); + + return getTimeRangeFromExpr(pCmd, pExpr->pRight); + } else { + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByNameEx(&pExpr->pLeft->colInfo, pCmd, &index) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_INVALID_SQL; + } + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); + SMeterMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; + + tSQLExpr* pRight = pExpr->pRight; + + TSKEY stime = 0; + TSKEY etime = INT64_MAX; + + if (getTimeRange(&stime, &etime, pRight, pExpr->nSQLOptr, pMeterMeta->precision) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg0); + return TSDB_CODE_INVALID_SQL; + } + + // update the timestamp query range + if (pCmd->stime < stime) { + pCmd->stime = stime; + } + + if (pCmd->etime > etime) { + pCmd->etime = etime; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t validateJoinExpr(SSqlCmd* pCmd, SCondExpr* pCondExpr) { + const char* msg1 = "super table join requires tags column"; + const char* msg2 = "timestamp join condition missing"; + const char* msg3 = "condition missing for join query"; + + if (!QUERY_IS_JOIN_QUERY(pCmd->type)) { + if (pCmd->numOfTables == 1) { + return TSDB_CODE_SUCCESS; + } else { + setErrMsg(pCmd, msg3); + return TSDB_CODE_INVALID_SQL; + } + } + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + if (UTIL_METER_IS_METRIC(pMeterMetaInfo)) { // for stable join, tag columns + // must be present for join + if (pCondExpr->pJoinExpr == NULL) { + setErrMsg(pCmd, msg1); + return TSDB_CODE_INVALID_SQL; + } + } + + if (!pCondExpr->tsJoin) { + setErrMsg(pCmd, msg2); return TSDB_CODE_INVALID_SQL; } - bool setTimeRange = false; + return TSDB_CODE_SUCCESS; +} + +static void cleanQueryExpr(SCondExpr* pCondExpr) { + if (pCondExpr->pTableCond) { + tSQLExprDestroy(pCondExpr->pTableCond); + } + + if (pCondExpr->pTagCond) { + tSQLExprDestroy(pCondExpr->pTagCond); + } + + if (pCondExpr->pColumnCond) { + tSQLExprDestroy(pCondExpr->pColumnCond); + } + + if (pCondExpr->pTimewindow) { + tSQLExprDestroy(pCondExpr->pTimewindow); + } + + if (pCondExpr->pJoinExpr) { + tSQLExprDestroy(pCondExpr->pJoinExpr); + } +} + +int32_t parseWhereClause(SSqlObj* pSql, tSQLExpr** pExpr) { + SSqlCmd* pCmd = &pSql->cmd; - /* tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space */ - char tmpTagCondBuf[TSDB_MAX_SQL_LEN] = {0}; - char* q = tmpTagCondBuf; + if (pExpr == NULL) { + return TSDB_CODE_SUCCESS; + } pCmd->stime = 0; pCmd->etime = INT64_MAX; - int32_t ret = getQueryCondExprImpl(pCmd, pExpr, &pCmd->stime, &pCmd->etime, &setTimeRange, &q); - if (ret != TSDB_CODE_SUCCESS) { + + int32_t ret = TSDB_CODE_SUCCESS; + + const char* msg1 = "invalid expression"; + SCondExpr condExpr = {0}; + + if ((*pExpr)->pLeft == NULL || (*pExpr)->pRight == NULL) { + setErrMsg(pCmd, msg1); return TSDB_CODE_INVALID_SQL; } - // query condition for tags - if (q == tmpTagCondBuf) { - pCmd->tagCond.len = 0; - } else { - int32_t qlen = (q - tmpTagCondBuf) + 1; - tmpTagCondBuf[qlen - 1] = 0; + ret = doParseWhereClause(pSql, pExpr, &condExpr); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } - if (pCmd->tagCond.type == TSQL_STABLE_QTYPE_SET) { - if (!UTIL_METER_IS_METRIC(pCmd)) { - setErrMsg(pCmd, msg2); - return TSDB_CODE_INVALID_SQL; - } + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + if (QUERY_IS_JOIN_QUERY(pCmd->type) && UTIL_METER_IS_METRIC(pMeterMetaInfo)) { + SColumnIndex index = {0}; - ret = setMetersIDForMetricQuery(pSql, tmpTagCondBuf); - } else { - if (pCmd->tagCond.allocSize < qlen + 1) { - pCmd->tagCond.allocSize = qlen + 1; - pCmd->tagCond.pData = realloc(pCmd->tagCond.pData, pCmd->tagCond.allocSize); + getColumnIndexByNameEx(&condExpr.pJoinExpr->pLeft->colInfo, pCmd, &index); + pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); + + int32_t columnInfo = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; + addRequiredTagColumn(pCmd, columnInfo, index.tableIndex); + + getColumnIndexByNameEx(&condExpr.pJoinExpr->pRight->colInfo, pCmd, &index); + pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); + + columnInfo = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; + addRequiredTagColumn(pCmd, columnInfo, index.tableIndex); + } + + cleanQueryExpr(&condExpr); + return ret; +} + +int32_t doParseWhereClause(SSqlObj* pSql, tSQLExpr** pExpr, SCondExpr* condExpr) { + const char* msg = "invalid filter expression"; + + int32_t type = 0; + SSqlCmd* pCmd = &pSql->cmd; + + /* + * tags query condition may be larger than 512bytes, + * therefore, we need to prepare enough large space + */ + char tableNameCond[TSDB_MAX_SQL_LEN] = {0}; + + int32_t ret = TSDB_CODE_SUCCESS; + if ((ret = getQueryCondExpr(pCmd, pExpr, condExpr, &type, (*pExpr)->nSQLOptr)) != TSDB_CODE_SUCCESS) { + return ret; + } + + doCompactQueryExpr(pExpr); + + // after expression compact, the expression tree is only include tag query condition + condExpr->pTagCond = (*pExpr); + + // 1. check if it is a join query + if ((ret = validateJoinExpr(pCmd, condExpr)) != TSDB_CODE_SUCCESS) { + return ret; + } + + // 2. get the query time range + if ((ret = getTimeRangeFromExpr(pCmd, condExpr->pTimewindow)) != TSDB_CODE_SUCCESS) { + return ret; + } + + // 3. get the tag query condition + if (condExpr->pTagCond != NULL) { + for (int32_t i = 0; i < pCmd->numOfTables; ++i) { + tSQLExpr* p1 = extractExprForSTable(pExpr, pCmd, i); + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, i); + + char c[TSDB_MAX_TAGS_LEN] = {0}; + char* str = c; + if ((ret = getTagCondString(pCmd, p1, &str)) != TSDB_CODE_SUCCESS) { + return ret; } - strcpy(pCmd->tagCond.pData, tmpTagCondBuf); - pCmd->tagCond.len = qlen; // plus one null-terminated symbol + tsSetMetricQueryCond(&pCmd->tagCond, pMeterMetaInfo->pMeterMeta->uid, c); + + doCompactQueryExpr(pExpr); + tSQLExprDestroy(p1); } - pCmd->tagCond.pData[pCmd->tagCond.len] = 0; + condExpr->pTagCond = NULL; + } + + // 4. get the table name query condition + if ((ret = getTablenameCond(pCmd, condExpr->pTableCond, tableNameCond)) != TSDB_CODE_SUCCESS) { + return ret; + } + + // 5. other column query condition + if ((ret = getColumnQueryCondInfo(pCmd, condExpr->pColumnCond, TK_AND)) != TSDB_CODE_SUCCESS) { + return ret; + } + + // 6. join condition + if ((ret = getJoinCondInfo(pSql, condExpr->pJoinExpr)) != TSDB_CODE_SUCCESS) { + return ret; } + // 7. query condition for table name + pCmd->tagCond.relType = (condExpr->relType == TK_AND) ? TSDB_RELATION_AND : TSDB_RELATION_OR; + ret = setTableCondForMetricQuery(pSql, condExpr->pTableCond, condExpr->tableCondIndex, tableNameCond); if (!validateFilterExpr(pCmd)) { - setErrMsg(pCmd, msg3); + setErrMsg(pCmd, msg); return TSDB_CODE_INVALID_SQL; } @@ -3197,10 +4301,14 @@ int32_t buildQueryCond(SSqlObj* pSql, tSQLExpr* pExpr) { } int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t optr, int16_t timePrecision) { + // this is join condition, do nothing + if (pRight->nSQLOptr == TK_ID) { + return TSDB_CODE_SUCCESS; + } + /* * filter primary ts filter expression like: - * "where ts in ('2015-12-12 4:8:12')" - * is not supported. + * where ts in ('2015-12-12 4:8:12') */ if (pRight->nSQLOptr == TK_SET || optr == TK_IN) { return TSDB_CODE_INVALID_SQL; @@ -3209,7 +4317,8 @@ int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t o int64_t val = 0; bool parsed = false; if (pRight->val.nType == TSDB_DATA_TYPE_BINARY) { - strdequote(pRight->val.pz); + pRight->val.nLen = strdequote(pRight->val.pz); + char* seg = strnchr(pRight->val.pz, '-', pRight->val.nLen, false); if (seg != NULL) { if (taosParseTime(pRight->val.pz, &val, pRight->val.nLen, TSDB_TIME_PRECISION_MICRO) == TSDB_CODE_SUCCESS) { @@ -3217,11 +4326,18 @@ int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t o } else { return TSDB_CODE_INVALID_SQL; } + } else { + SSQLToken token = {.z = pRight->val.pz, .n = pRight->val.nLen, .type = TK_ID}; + int32_t len = tSQLGetToken(pRight->val.pz, &token.type); + + if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->val.nLen) { + return TSDB_CODE_INVALID_SQL; + } } } else if (pRight->nSQLOptr == TK_INTEGER && timePrecision == TSDB_TIME_PRECISION_MILLI) { /* * if the pRight->nSQLOptr == TK_INTEGER/TK_FLOAT, the value is adaptive, we - * need the time precision of metermeta to transfer the value in MICROSECOND + * need the time precision in metermeta to transfer the value in MICROSECOND * * Additional check to avoid data overflow */ @@ -3274,15 +4390,19 @@ int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t o } int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd) { + const char rep[] = {'(', ')', '*', ',', '.', '/', '\\', '+', '-', '%', ' '}; + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { char* fieldName = tscFieldInfoGetField(pCmd, i)->name; for (int32_t j = 0; j < TSDB_COL_NAME_LEN && fieldName[j] != 0; ++j) { - if (fieldName[j] == '(' || fieldName[j] == ')' || fieldName[j] == '*' || fieldName[j] == ',' || - fieldName[j] == '.' || fieldName[j] == '/' || fieldName[j] == '+' || fieldName[j] == '-' || - fieldName[j] == ' ') { - fieldName[j] = '_'; + for (int32_t k = 0; k < tListLen(rep); ++k) { + if (fieldName[j] == rep[k]) { + fieldName[j] = '_'; + break; + } } } + fieldName[TSDB_COL_NAME_LEN - 1] = 0; } @@ -3301,7 +4421,7 @@ int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd) { return TSDB_CODE_SUCCESS; } -int32_t setFillPolicy(SSqlCmd* pCmd, SQuerySQL* pQuerySQL) { +int32_t parseFillClause(SSqlCmd* pCmd, SQuerySQL* pQuerySQL) { tVariantList* pFillToken = pQuerySQL->fillType; tVariantListItem* pItem = &pFillToken->a[0]; @@ -3391,6 +4511,7 @@ int32_t setFillPolicy(SSqlCmd* pCmd, SQuerySQL* pQuerySQL) { static void setDefaultOrderInfo(SSqlCmd* pCmd) { /* set default timestamp order information for all queries */ pCmd->order.order = TSQL_SO_ASC; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); if (isTopBottomQuery(pCmd)) { pCmd->order.order = TSQL_SO_ASC; @@ -3400,32 +4521,40 @@ static void setDefaultOrderInfo(SSqlCmd* pCmd) { } /* for metric query, set default ascending order for group output */ - if (UTIL_METER_IS_METRIC(pCmd)) { + if (UTIL_METER_IS_METRIC(pMeterMetaInfo)) { pCmd->groupbyExpr.orderType = TSQL_SO_ASC; } } -int32_t setOrderByClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql, SSchema* pSchema, int32_t numOfCols) { - const char* msg = "only support order by primary timestamp"; - const char* msg3 = "invalid column name"; - const char* msg5 = "only support order by primary timestamp and queried column"; - const char* msg6 = "only support order by primary timestamp and first tag in groupby clause"; +int32_t parseOrderbyClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql, SSchema* pSchema, int32_t numOfCols) { + const char* msg0 = "only support order by primary timestamp"; + const char* msg1 = "invalid column name"; + const char* msg2 = "only support order by primary timestamp and queried column"; + const char* msg3 = "only support order by primary timestamp and first tag in groupby clause"; setDefaultOrderInfo(pCmd); + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); if (pQuerySql->pSortOrder == NULL) { return TSDB_CODE_SUCCESS; } tVariantList* pSortorder = pQuerySql->pSortOrder; - if (UTIL_METER_IS_NOMRAL_METER(pCmd)) { + + /* + * for table query, there is only one or none order option is allowed, which is the + * ts or values(top/bottom) order is supported. + * + * for super table query, the order option must be less than 3. + */ + if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { if (pSortorder->nExpr > 1) { - setErrMsg(pCmd, msg); + setErrMsg(pCmd, msg0); return TSDB_CODE_INVALID_SQL; } } else { if (pSortorder->nExpr > 2) { - setErrMsg(pCmd, msg6); + setErrMsg(pCmd, msg3); return TSDB_CODE_INVALID_SQL; } } @@ -3438,26 +4567,34 @@ int32_t setOrderByClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql, SSchema* pSchema, return TSDB_CODE_SUCCESS; } - SSQLToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; + SSQLToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; + SColumnIndex index = {0}; + + if (UTIL_METER_IS_METRIC(pMeterMetaInfo)) { // metric query + if (getColumnIndexByNameEx(&columnName, pCmd, &index) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg1); + return TSDB_CODE_INVALID_SQL; + } + + bool orderByTags = false; + bool orderByTS = false; + bool orderByCol = false; - if (UTIL_METER_IS_METRIC(pCmd)) { // metric query - SSchema* pTagSchema = tsGetTagSchema(pCmd->pMeterMeta); - int32_t columnIndex = getColumnIndexByName(&columnName, pTagSchema, pCmd->pMeterMeta->numOfTags); - bool orderByTags = false; - bool orderByTS = false; - if (pCmd->groupbyExpr.tagIndex[0] == columnIndex) { - if (columnIndex >= 0 || (strncasecmp(columnName.z, TSQL_TBNAME_L, 6) == 0)) { + if (index.columnIndex >= pMeterMetaInfo->pMeterMeta->numOfColumns) { + int32_t relTagIndex = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; + if (relTagIndex == pCmd->groupbyExpr.columnInfo[0].colIdx) { orderByTags = true; } + } else if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + orderByTags = true; } - columnIndex = getColumnIndexByName(&columnName, pSchema, numOfCols); - if (PRIMARYKEY_TIMESTAMP_COL_INDEX == columnIndex) { + if (PRIMARYKEY_TIMESTAMP_COL_INDEX == index.columnIndex) { orderByTS = true; } - if (!(orderByTags || orderByTS)) { - setErrMsg(pCmd, msg6); + if (!(orderByTags || orderByTS) && !isTopBottomQuery(pCmd)) { + setErrMsg(pCmd, msg3); return TSDB_CODE_INVALID_SQL; } else { assert(!(orderByTags && orderByTS)); @@ -3465,8 +4602,22 @@ int32_t setOrderByClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql, SSchema* pSchema, if (pSortorder->nExpr == 1) { if (orderByTags) { - pCmd->groupbyExpr.orderIdx = columnIndex; + pCmd->groupbyExpr.orderIndex = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; pCmd->groupbyExpr.orderType = pQuerySql->pSortOrder->a[0].sortOrder; + } else if (isTopBottomQuery(pCmd)) { + /* order of top/bottom query in interval is not valid */ + SSqlExpr* pExpr = tscSqlExprGet(pCmd, 0); + assert(pExpr->functionId == TSDB_FUNC_TS); + + pExpr = tscSqlExprGet(pCmd, 1); + if (pExpr->colInfo.colIdx != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + setErrMsg(pCmd, msg2); + return TSDB_CODE_INVALID_SQL; + } + + pCmd->order.order = pQuerySql->pSortOrder->a[0].sortOrder; + pCmd->order.orderColId = pSchema[index.columnIndex].colId; + return TSDB_CODE_SUCCESS; } else { pCmd->order.order = pSortorder->a[0].sortOrder; pCmd->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; @@ -3474,11 +4625,23 @@ int32_t setOrderByClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql, SSchema* pSchema, } if (pSortorder->nExpr == 2) { + if (orderByTags) { + pCmd->groupbyExpr.orderIndex = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; + pCmd->groupbyExpr.orderType = pQuerySql->pSortOrder->a[0].sortOrder; + } else { + pCmd->order.order = pSortorder->a[0].sortOrder; + pCmd->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; + } + tVariant* pVar2 = &pSortorder->a[1].pVar; SSQLToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; - columnIndex = getColumnIndexByName(&cname, pSchema, numOfCols); - if (columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - setErrMsg(pCmd, msg5); + if (getColumnIndexByNameEx(&cname, pCmd, &index) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg1); + return TSDB_CODE_INVALID_SQL; + } + + if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + setErrMsg(pCmd, msg2); return TSDB_CODE_INVALID_SQL; } else { pCmd->order.order = pSortorder->a[1].sortOrder; @@ -3487,30 +4650,29 @@ int32_t setOrderByClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql, SSchema* pSchema, } } else { // meter query - int32_t columnIndex = getColumnIndexByName(&columnName, pSchema, numOfCols); - if (columnIndex <= -1) { - setErrMsg(pCmd, msg3); + if (getColumnIndexByNameEx(&columnName, pCmd, &index) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg1); return TSDB_CODE_INVALID_SQL; } - if (columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX && !isTopBottomQuery(pCmd)) { - setErrMsg(pCmd, msg5); + if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX && !isTopBottomQuery(pCmd)) { + setErrMsg(pCmd, msg2); return TSDB_CODE_INVALID_SQL; } - if (isTopBottomQuery(pCmd) && pCmd->nAggTimeInterval >= 0) { + if (isTopBottomQuery(pCmd)) { /* order of top/bottom query in interval is not valid */ SSqlExpr* pExpr = tscSqlExprGet(pCmd, 0); - assert(pExpr->sqlFuncId == TSDB_FUNC_TS); + assert(pExpr->functionId == TSDB_FUNC_TS); pExpr = tscSqlExprGet(pCmd, 1); - if (pExpr->colInfo.colIdx != columnIndex && columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - setErrMsg(pCmd, msg); + if (pExpr->colInfo.colIdx != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + setErrMsg(pCmd, msg2); return TSDB_CODE_INVALID_SQL; } pCmd->order.order = pQuerySql->pSortOrder->a[0].sortOrder; - pCmd->order.orderColId = pSchema[columnIndex].colId; + pCmd->order.orderColId = pSchema[index.columnIndex].colId; return TSDB_CODE_SUCCESS; } @@ -3521,7 +4683,10 @@ int32_t setOrderByClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql, SSchema* pSchema, } int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { - SSqlCmd* pCmd = &pSql->cmd; + const int32_t DEFAULT_TABLE_INDEX = 0; + + SSqlCmd* pCmd = &pSql->cmd; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, DEFAULT_TABLE_INDEX); SAlterTableSQL* pAlterSQL = pInfo->pAlterInfo; pCmd->command = TSDB_SQL_ALTER_TABLE; @@ -3532,33 +4697,33 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - if (setMeterID(pSql, &(pAlterSQL->name)) != TSDB_CODE_SUCCESS) { + if (setMeterID(pSql, &(pAlterSQL->name), 0) != TSDB_CODE_SUCCESS) { const char* msg = "table name too long"; setErrMsg(pCmd, msg); return TSDB_CODE_INVALID_SQL; } - int32_t ret = tscGetMeterMeta(pSql, pCmd->name); + int32_t ret = tscGetMeterMeta(pSql, pMeterMetaInfo->name, DEFAULT_TABLE_INDEX); if (ret != TSDB_CODE_SUCCESS) { return ret; } - SMeterMeta* pMeterMeta = pCmd->pMeterMeta; + SMeterMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; SSchema* pSchema = tsGetSchema(pMeterMeta); if (pInfo->sqlType == ALTER_TABLE_TAGS_ADD || pInfo->sqlType == ALTER_TABLE_TAGS_DROP || pInfo->sqlType == ALTER_TABLE_TAGS_CHG) { - if (UTIL_METER_IS_NOMRAL_METER(pCmd)) { + if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { const char* msg = "manipulation of tag available for metric"; setErrMsg(pCmd, msg); return TSDB_CODE_INVALID_SQL; } - } else if ((pInfo->sqlType == ALTER_TABLE_TAGS_SET) && (UTIL_METER_IS_METRIC(pCmd))) { + } else if ((pInfo->sqlType == ALTER_TABLE_TAGS_SET) && (UTIL_METER_IS_METRIC(pMeterMetaInfo))) { const char* msg = "set tag value only available for table"; setErrMsg(pCmd, msg); return TSDB_CODE_INVALID_SQL; } else if ((pInfo->sqlType == ALTER_TABLE_ADD_COLUMN || pInfo->sqlType == ALTER_TABLE_DROP_COLUMN) && - UTIL_METER_IS_CREATE_FROM_METRIC(pCmd)) { + UTIL_METER_IS_CREATE_FROM_METRIC(pMeterMetaInfo)) { const char* msg = "column can only be modified by metric"; setErrMsg(pCmd, msg); return TSDB_CODE_INVALID_SQL; @@ -3656,26 +4821,16 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - bool srcFound = false; - bool dstFound = false; - for (int32_t i = 0; i < pMeterMeta->numOfTags; ++i) { - int32_t tagIdx = i + pMeterMeta->numOfColumns; - char* tagName = pSchema[tagIdx].name; - - size_t nameLen = strlen(tagName); - if ((!srcFound) && (strncasecmp(tagName, pSrcItem->pVar.pz, nameLen) == 0 && (pSrcItem->pVar.nLen == nameLen))) { - srcFound = true; - } + SColumnIndex srcIndex = COLUMN_INDEX_INITIALIZER; + SColumnIndex destIndex = COLUMN_INDEX_INITIALIZER; - // todo extract method - if ((!dstFound) && (strncasecmp(tagName, pDstItem->pVar.pz, nameLen) == 0 && (pDstItem->pVar.nLen == nameLen))) { - dstFound = true; - } + SSQLToken srcToken = {.z = pSrcItem->pVar.pz, .n = pSrcItem->pVar.nLen, .type = TK_STRING}; + if (getColumnIndexByNameEx(&srcToken, pCmd, &srcIndex) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_INVALID_SQL; } - if ((!srcFound) || dstFound) { - const char* msg = "invalid tag name"; - setErrMsg(pCmd, msg); + SSQLToken destToken = {.z = pDstItem->pVar.pz, .n = pDstItem->pVar.nLen, .type = TK_STRING}; + if (getColumnIndexByNameEx(&destToken, pCmd, &destIndex) == TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } @@ -3686,47 +4841,51 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { memset(name, 0, tListLen(name)); strncpy(name, pVarList->a[1].pVar.pz, pVarList->a[1].pVar.nLen); tscFieldInfoSetValue(&pCmd->fieldsInfo, 1, TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); - pCmd->numOfCols = 2; + pCmd->numOfCols = 2; } else if (pInfo->sqlType == ALTER_TABLE_TAGS_SET) { + const char* msg0 = "tag name too long"; + const char* msg1 = "invalid tag value"; + const char* msg2 = "invalid tag name"; + const char* msg3 = "tag value too long"; + pCmd->count = TSDB_ALTER_TABLE_UPDATE_TAG_VAL; // Note: update can only be applied to meter not metric. // the following is handle display tags value for meters created according to metric - char* pTagValue = tsGetTagsValue(pCmd->pMeterMeta); tVariantList* pVarList = pAlterSQL->varList; tVariant* pTagName = &pVarList->a[0].pVar; if (pTagName->nLen > TSDB_COL_NAME_LEN) { - const char* msg = "tag name too long"; - setErrMsg(pCmd, msg); + setErrMsg(pCmd, msg0); return TSDB_CODE_INVALID_SQL; } int32_t tagsIndex = -1; - SSchema* pTagsSchema = tsGetTagSchema(pCmd->pMeterMeta); - for (int32_t i = 0; i < pCmd->pMeterMeta->numOfTags; ++i) { + SSchema* pTagsSchema = tsGetTagSchema(pMeterMetaInfo->pMeterMeta); + + for (int32_t i = 0; i < pMeterMetaInfo->pMeterMeta->numOfTags; ++i) { if (strcmp(pTagName->pz, pTagsSchema[i].name) == 0 && strlen(pTagsSchema[i].name) == pTagName->nLen) { tagsIndex = i; - tVariantDump(&pVarList->a[1].pVar, pCmd->payload, pTagsSchema[i].type); break; } - - pTagValue += pTagsSchema[i].bytes; } if (tagsIndex == -1) { - const char* msg = "invalid tag name"; - setErrMsg(pCmd, msg); + setErrMsg(pCmd, msg2); + return TSDB_CODE_INVALID_SQL; + } + + if (tVariantDump(&pVarList->a[1].pVar, pCmd->payload, pTagsSchema[tagsIndex].type) != TSDB_CODE_SUCCESS) { + setErrMsg(pCmd, msg1); return TSDB_CODE_INVALID_SQL; } // validate the length of binary - if (pTagsSchema[tagsIndex].type == TSDB_DATA_TYPE_BINARY && + if ((pTagsSchema[tagsIndex].type == TSDB_DATA_TYPE_BINARY || pTagsSchema[tagsIndex].type == TSDB_DATA_TYPE_NCHAR) && pVarList->a[1].pVar.nLen > pTagsSchema[tagsIndex].bytes) { - const char* msg = "tag value too long"; - setErrMsg(pCmd, msg); + setErrMsg(pCmd, msg3); return TSDB_CODE_INVALID_SQL; } @@ -3751,7 +4910,6 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { tscFieldInfoSetValFromField(&pCmd->fieldsInfo, 0, &pFieldList->p[0]); pCmd->numOfCols = 1; // only one column - } else if (pInfo->sqlType == ALTER_TABLE_DROP_COLUMN) { pCmd->count = TSDB_ALTER_TABLE_DROP_COLUMN; @@ -3816,7 +4974,7 @@ int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd) { } for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { - int32_t functId = tscSqlExprGet(pCmd, i)->sqlFuncId; + int32_t functId = tscSqlExprGet(pCmd, i)->functionId; if (!IS_STREAM_QUERY_VALID(aAggs[functId].nStatus)) { setErrMsg(pCmd, msg1); return TSDB_CODE_INVALID_SQL; @@ -3833,12 +4991,13 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd) { // multi-output set/ todo refactor for (int32_t k = 0; k < pCmd->fieldsInfo.numOfOutputCols; ++k) { SSqlExpr* pExpr = tscSqlExprGet(pCmd, k); - if (pExpr->sqlFuncId == TSDB_FUNC_PRJ || pExpr->sqlFuncId == TSDB_FUNC_TAGPRJ || - pExpr->sqlFuncId == TSDB_FUNC_DIFF || pExpr->sqlFuncId == TSDB_FUNC_ARITHM) { + if (pExpr->functionId == TSDB_FUNC_PRJ || pExpr->functionId == TSDB_FUNC_DIFF || + pExpr->functionId == TSDB_FUNC_ARITHM) { isProjectionFunction = true; } } - if (pCmd->metricQuery == 0 || isProjectionFunction == true) { + + if (isProjectionFunction) { setErrMsg(pCmd, msg); } @@ -3855,11 +5014,11 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_INVALID_SQL; } - SDNodeDynConfOption DNODE_DYNAMIC_CFG_OPTIONS[13] = { + SDNodeDynConfOption DNODE_DYNAMIC_CFG_OPTIONS[14] = { {"resetLog", 8}, {"resetQueryCache", 15}, {"dDebugFlag", 10}, {"rpcDebugFlag", 12}, {"tmrDebugFlag", 12}, {"cDebugFlag", 10}, {"uDebugFlag", 10}, {"mDebugFlag", 10}, {"sdbDebugFlag", 12}, {"httpDebugFlag", 13}, {"monitorDebugFlag", 16}, {"qDebugflag", 10}, - {"debugFlag", 9}}; + {"debugFlag", 9}, {"monitor", 7}}; SSQLToken* pOptionToken = &pOptions->a[1]; @@ -3871,6 +5030,14 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_SUCCESS; } } + } else if ((strncasecmp(DNODE_DYNAMIC_CFG_OPTIONS[13].name, pOptionToken->z, pOptionToken->n) == 0) && + (DNODE_DYNAMIC_CFG_OPTIONS[13].len == pOptionToken->n)) { + SSQLToken* pValToken = &pOptions->a[2]; + int32_t val = strtol(pValToken->z, NULL, 10); + if (val != 0 && val != 1) { + return TSDB_CODE_INVALID_SQL; // options value is invalid + } + return TSDB_CODE_SUCCESS; } else { SSQLToken* pValToken = &pOptions->a[2]; @@ -3880,7 +5047,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_INVALID_SQL; } - for (int32_t i = 2; i < tListLen(DNODE_DYNAMIC_CFG_OPTIONS); ++i) { + for (int32_t i = 2; i < tListLen(DNODE_DYNAMIC_CFG_OPTIONS) - 1; ++i) { SDNodeDynConfOption* pOption = &DNODE_DYNAMIC_CFG_OPTIONS[i]; if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { @@ -3893,15 +5060,51 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_INVALID_SQL; } +int32_t validateLocalConfig(tDCLSQL* pOptions) { + if (pOptions->nTokens < 1 || pOptions->nTokens > 2) { + return TSDB_CODE_INVALID_SQL; + } + + SDNodeDynConfOption LOCAL_DYNAMIC_CFG_OPTIONS[6] = {{"resetLog", 8}, {"rpcDebugFlag", 12}, {"tmrDebugFlag", 12}, + {"cDebugFlag", 10}, {"uDebugFlag", 10}, {"debugFlag", 9}}; + + SSQLToken* pOptionToken = &pOptions->a[0]; + + if (pOptions->nTokens == 1) { + // reset log does not need value + for (int32_t i = 0; i < 1; ++i) { + SDNodeDynConfOption* pOption = &LOCAL_DYNAMIC_CFG_OPTIONS[i]; + if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { + return TSDB_CODE_SUCCESS; + } + } + } else { + SSQLToken* pValToken = &pOptions->a[1]; + + int32_t val = strtol(pValToken->z, NULL, 10); + if (val < 131 || val > 199) { + // options value is out of valid range + return TSDB_CODE_INVALID_SQL; + } + + for (int32_t i = 1; i < tListLen(LOCAL_DYNAMIC_CFG_OPTIONS); ++i) { + SDNodeDynConfOption* pOption = &LOCAL_DYNAMIC_CFG_OPTIONS[i]; + if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { + // options is valid + return TSDB_CODE_SUCCESS; + } + } + } + return TSDB_CODE_INVALID_SQL; +} + int32_t validateColumnName(char* name) { bool ret = isKeyWord(name, strlen(name)); if (ret) { return TSDB_CODE_INVALID_SQL; } - SSQLToken token = { - .z = name, - }; + SSQLToken token = {.z = name}; token.n = tSQLGetToken(name, &token.type); if (token.type != TK_STRING && token.type != TK_ID) { @@ -3938,6 +5141,7 @@ bool hasTimestampForPointInterpQuery(SSqlCmd* pCmd) { int32_t parseLimitClause(SSqlObj* pSql, SQuerySQL* pQuerySql) { SSqlCmd* pCmd = &pSql->cmd; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); const char* msg0 = "soffset/offset can not be less than 0"; const char* msg1 = "slimit/soffset only available for STable query"; @@ -3946,9 +5150,9 @@ int32_t parseLimitClause(SSqlObj* pSql, SQuerySQL* pQuerySql) { // handle the limit offset value, validate the limit pCmd->limit = pQuerySql->limit; - pCmd->glimit = pQuerySql->glimit; + pCmd->slimit = pQuerySql->slimit; - if (pCmd->glimit.offset < 0 || pCmd->limit.offset < 0) { + if (pCmd->slimit.offset < 0 || pCmd->limit.offset < 0) { setErrMsg(pCmd, msg0); return TSDB_CODE_INVALID_SQL; } @@ -3958,7 +5162,7 @@ int32_t parseLimitClause(SSqlObj* pSql, SQuerySQL* pQuerySql) { pCmd->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; } - if (UTIL_METER_IS_METRIC(pCmd)) { + if (UTIL_METER_IS_METRIC(pMeterMetaInfo)) { bool queryOnTags = false; int32_t ret = tscQueryOnlyMetricTags(pCmd, &queryOnTags); if (ret != TSDB_CODE_SUCCESS) { @@ -3968,13 +5172,13 @@ int32_t parseLimitClause(SSqlObj* pSql, SQuerySQL* pQuerySql) { if (queryOnTags == true) { // local handle the metric tag query pCmd->command = TSDB_SQL_RETRIEVE_TAGS; } else { - if (tscProjectionQueryOnMetric(pSql) && (pCmd->glimit.limit > 0 || pCmd->glimit.offset > 0)) { + if (tscProjectionQueryOnMetric(pCmd) && (pCmd->slimit.limit > 0 || pCmd->slimit.offset > 0)) { setErrMsg(pCmd, msg3); return TSDB_CODE_INVALID_SQL; } } - if (pCmd->glimit.limit == 0) { + if (pCmd->slimit.limit == 0) { tscTrace("%p limit 0, no output result", pSql); pCmd->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; return TSDB_CODE_SUCCESS; @@ -3985,15 +5189,14 @@ int32_t parseLimitClause(SSqlObj* pSql, SQuerySQL* pQuerySql) { * created according to this super table from management node. * And then launching multiple async-queries on required virtual nodes, which is the first-stage query operation. */ - int32_t code = tscGetMetricMeta(pSql, pCmd->name); + int32_t code = tscGetMetricMeta(pSql); if (code != TSDB_CODE_SUCCESS) { return code; } - //No tables included. No results generated. Query results are empty. - SMetricMeta* pMetricMeta = pCmd->pMetricMeta; - if (pCmd->pMeterMeta == NULL || pMetricMeta == NULL || pMetricMeta->numOfVnodes == 0 || - pMetricMeta->numOfMeters == 0) { + // No tables included. No results generated. Query results are empty. + SMetricMeta* pMetricMeta = pMeterMetaInfo->pMetricMeta; + if (pMeterMetaInfo->pMeterMeta == NULL || pMetricMeta == NULL || pMetricMeta->numOfMeters == 0) { tscTrace("%p no table in metricmeta, no output result", pSql); pCmd->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; } @@ -4001,7 +5204,7 @@ int32_t parseLimitClause(SSqlObj* pSql, SQuerySQL* pQuerySql) { // keep original limitation value in globalLimit pCmd->globalLimit = pCmd->limit.limit; } else { - if (pCmd->glimit.limit != -1 || pCmd->glimit.offset != 0) { + if (pCmd->slimit.limit != -1 || pCmd->slimit.offset != 0) { setErrMsg(pCmd, msg1); return TSDB_CODE_INVALID_SQL; } @@ -4009,7 +5212,7 @@ int32_t parseLimitClause(SSqlObj* pSql, SQuerySQL* pQuerySql) { // filter the query functions operating on "tbname" column that are not supported by normal columns. for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); - if (pExpr->colInfo.colIdx == -1) { + if (pExpr->colInfo.colIdx == TSDB_TBNAME_COLUMN_INDEX) { setErrMsg(pCmd, msg2); return TSDB_CODE_INVALID_SQL; } @@ -4088,3 +5291,394 @@ int32_t parseCreateDBOptions(SCreateDBInfo* pCreateDbSql, SSqlCmd* pCmd) { return TSDB_CODE_SUCCESS; } + +void tscAddTimestampColumn(SSqlCmd* pCmd, int16_t functionId, int16_t tableIndex) { + // the first column not timestamp column, add it + SSqlExpr* pExpr = NULL; + if (pCmd->exprsInfo.numOfExprs > 0) { + pExpr = tscSqlExprGet(pCmd, 0); + } + + if (pExpr == NULL || pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX || pExpr->functionId != functionId) { + SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + + pExpr = tscSqlExprInsert(pCmd, 0, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); + pExpr->colInfo.flag = TSDB_COL_NORMAL; + + // NOTE: tag column does not add to source column list + SColumnList ids = getColumnList(1, tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX); + + insertResultField(pCmd, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, "ts"); + } +} + +void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t tableIndex) { + if (pParentObj->cmd.groupbyExpr.numOfGroupCols > 0) { + int32_t num = pSql->cmd.exprsInfo.numOfExprs; + SSqlExpr* pExpr = tscSqlExprGet(&pSql->cmd, num - 1); + SSqlCmd* pCmd = &pSql->cmd; + + if (pExpr->functionId != TSDB_FUNC_TAG) { + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0); + int16_t columnInfo = tscGetJoinTagColIndexByUid(pCmd, pMeterMetaInfo->pMeterMeta->uid); + SColumnIndex index = {.tableIndex = 0, .columnIndex = columnInfo}; + SSchema* pSchema = tsGetTagSchema(pMeterMetaInfo->pMeterMeta); + + int16_t type = pSchema[index.columnIndex].type; + int16_t bytes = pSchema[index.columnIndex].bytes; + char* name = pSchema[index.columnIndex].name; + + pExpr = tscSqlExprInsert(pCmd, pCmd->fieldsInfo.numOfOutputCols, TSDB_FUNC_TAG, &index, type, bytes, bytes); + pExpr->colInfo.flag = TSDB_COL_TAG; + + // NOTE: tag column does not add to source column list + SColumnList ids = {0}; + insertResultField(pCmd, pCmd->fieldsInfo.numOfOutputCols, &ids, bytes, type, name); + + int32_t relIndex = index.columnIndex; + + pExpr->colInfo.colIdx = relIndex; + pCmd->groupbyExpr.columnInfo[0].colIdx = relIndex; + + addRequiredTagColumn(pCmd, pCmd->groupbyExpr.columnInfo[0].colIdx, 0); + } + } +} + +void doAddGroupColumnForSubquery(SSqlCmd* pCmd, int32_t tagIndex) { + int32_t index = pCmd->groupbyExpr.columnInfo[tagIndex].colIdx; + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + + SSchema* pSchema = tsGetColumnSchema(pMeterMetaInfo->pMeterMeta, index); + SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = index}; + + SSqlExpr* pExpr = tscSqlExprInsert(pCmd, pCmd->fieldsInfo.numOfOutputCols, TSDB_FUNC_PRJ, &colIndex, pSchema->type, + pSchema->bytes, pSchema->bytes); + + pExpr->colInfo.flag = TSDB_COL_NORMAL; + pExpr->param[0].i64Key = 1; + pExpr->numOfParams = 1; + + // NOTE: tag column does not add to source column list + SColumnList list = {0}; + list.num = 1; + list.ids[0] = colIndex; + + insertResultField(pCmd, pCmd->fieldsInfo.numOfOutputCols, &list, pSchema->bytes, pSchema->type, pSchema->name); + tscFieldInfoUpdateVisible(&pCmd->fieldsInfo, pCmd->fieldsInfo.numOfOutputCols - 1, false); +} + +static void doUpdateSqlFunctionForTagPrj(SSqlCmd* pCmd) { + int32_t tagLength = 0; + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); + if (pExpr->functionId == TSDB_FUNC_TAGPRJ || pExpr->functionId == TSDB_FUNC_TAG) { + pExpr->functionId = TSDB_FUNC_TAG_DUMMY; + tagLength += pExpr->resBytes; + } else if (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + pExpr->functionId = TSDB_FUNC_TS_DUMMY; + tagLength += pExpr->resBytes; + } + } + + int16_t resType = 0; + int16_t resBytes = 0; + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); + if (pExpr->functionId != TSDB_FUNC_TAG_DUMMY && pExpr->functionId != TSDB_FUNC_TS_DUMMY) { + SSchema* pColSchema = &pSchema[pExpr->colInfo.colIdx]; + getResultDataInfo(pColSchema->type, pColSchema->bytes, pExpr->functionId, pExpr->param[0].i64Key, &pExpr->resType, + &pExpr->resBytes, &pExpr->interResBytes, tagLength, true); + } + } +} + +static bool tagColumnInGroupby(SSqlGroupbyExpr* pGroupbyExpr, int16_t columnId) { + for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { + if (columnId == pGroupbyExpr->columnInfo[j].colId && pGroupbyExpr->columnInfo[j].flag == TSDB_COL_TAG) { + return true; + } + } + + return false; +} + +static bool onlyTagPrjFunction(SSqlCmd* pCmd) { + bool hasTagPrj = false; + bool hasColumnPrj = false; + + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); + if (pExpr->functionId == TSDB_FUNC_PRJ) { + hasColumnPrj = true; + } else if (pExpr->functionId == TSDB_FUNC_TAGPRJ) { + hasTagPrj = true; + } + } + + return (hasTagPrj) && (hasColumnPrj == false); +} + +// check if all the tags prj columns belongs to the group by columns +static bool allTagPrjInGroupby(SSqlCmd* pCmd) { + bool allInGroupby = true; + + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); + if (pExpr->functionId != TSDB_FUNC_TAGPRJ) { + continue; + } + + if (!tagColumnInGroupby(&pCmd->groupbyExpr, pExpr->colInfo.colId)) { + allInGroupby = false; + break; + } + } + + // all selected tag columns belong to the group by columns set, always correct + return allInGroupby; +} + +static void updateTagPrjFunction(SSqlCmd* pCmd) { + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); + if (pExpr->functionId == TSDB_FUNC_TAGPRJ) { + pExpr->functionId = TSDB_FUNC_TAG; + } + } +} + +/* + * check for selectivity function + tags column function both exist. + * 1. tagprj functions are not compatible with aggregated function when missing "group by" clause + * 2. if selectivity function and tagprj function both exist, there should be only + * one selectivity function exists. + */ +static int32_t checkUpdateTagPrjFunctions(SSqlCmd* pCmd) { + const char* msg1 = "only one selectivity function allowed in presence of tags function"; + const char* msg2 = "functions not allowed"; + + bool tagColExists = false; + int16_t numOfTimestamp = 0; // primary timestamp column + int16_t numOfSelectivity = 0; + int16_t numOfAggregation = 0; + + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); + if (pExpr->functionId == TSDB_FUNC_TAGPRJ || + (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)) { + tagColExists = true; + break; + } + } + + if (tagColExists) { // check if the selectivity function exists + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + int16_t functionId = tscSqlExprGet(pCmd, i)->functionId; + if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TS) { + continue; + } + + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + numOfSelectivity++; + } else { + numOfAggregation++; + } + } + + // When the tag projection function on tag column that is not in the group by clause, aggregation function and + // selectivity function exist in select clause is not allowed. + if(numOfAggregation > 0) { + setErrMsg(pCmd, msg1); + return TSDB_CODE_INVALID_SQL; + } + + /* + * if numOfSelectivity equals to 0, it is a super table projection query + */ + if (numOfSelectivity == 1) { + doUpdateSqlFunctionForTagPrj(pCmd); + } else if (numOfSelectivity > 1) { + /* + * If more than one selectivity functions exist, all the selectivity functions must be last_row. + * Otherwise, return with error code. + */ + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + int16_t functionId = tscSqlExprGet(pCmd, i)->functionId; + if (functionId == TSDB_FUNC_TAGPRJ) { + continue; + } + + if (((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) && (functionId != TSDB_FUNC_LAST_ROW)) { + setErrMsg(pCmd, msg1); + return TSDB_CODE_INVALID_SQL; + } + } + + doUpdateSqlFunctionForTagPrj(pCmd); + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd) { + const char* msg2 = "interval not allowed in group by normal column"; + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + + SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + int16_t bytes = 0; + int16_t type = 0; + char* name = NULL; + + for (int32_t i = 0; i < pCmd->groupbyExpr.numOfGroupCols; ++i) { + SColIndexEx* pColIndex = &pCmd->groupbyExpr.columnInfo[i]; + + int16_t colIndex = pColIndex->colIdx; + if (pColIndex->colIdx == TSDB_TBNAME_COLUMN_INDEX) { + type = TSDB_DATA_TYPE_BINARY; + bytes = TSDB_METER_NAME_LEN; + name = TSQL_TBNAME_L; + } else { + colIndex = (TSDB_COL_IS_TAG(pColIndex->flag)) ? pMeterMetaInfo->pMeterMeta->numOfColumns + pColIndex->colIdx + : pColIndex->colIdx; + + type = pSchema[colIndex].type; + bytes = pSchema[colIndex].bytes; + name = pSchema[colIndex].name; + } + + if (TSDB_COL_IS_TAG(pColIndex->flag)) { + SColumnIndex index = {.tableIndex = pCmd->groupbyExpr.tableIndex, .columnIndex = colIndex}; + + SSqlExpr* pExpr = + tscSqlExprInsert(pCmd, pCmd->fieldsInfo.numOfOutputCols, TSDB_FUNC_TAG, &index, type, bytes, bytes); + + pExpr->colInfo.flag = TSDB_COL_TAG; + + // NOTE: tag column does not add to source column list + SColumnList ids = {0}; + insertResultField(pCmd, pCmd->fieldsInfo.numOfOutputCols, &ids, bytes, type, name); + } else { + // if this query is "group by" normal column, interval is not allowed + if (pCmd->nAggTimeInterval > 0) { + setErrMsg(pCmd, msg2); + return TSDB_CODE_INVALID_SQL; + } + + bool hasGroupColumn = false; + for (int32_t j = 0; j < pCmd->fieldsInfo.numOfOutputCols; ++j) { + SSqlExpr* pExpr = tscSqlExprGet(pCmd, j); + if (pExpr->colInfo.colId == pColIndex->colId) { + break; + } + } + + /* + * if the group by column does not required by user, add this column into the final result set + * but invisible to user + */ + if (!hasGroupColumn) { + doAddGroupColumnForSubquery(pCmd, i); + } + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t doFunctionsCompatibleCheck(SSqlObj* pSql) { + const char* msg1 = "functions/columns not allowed in group by query"; + const char* msg2 = "interval not allowed in group by normal column"; + const char* msg3 = "group by not allowed on projection query"; + const char* msg4 = "tags retrieve not compatible with group by"; + const char* msg5 = "retrieve tags not compatible with group by "; + + SSqlCmd* pCmd = &pSql->cmd; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + + // only retrieve tags, group by is not supportted + if (pCmd->command == TSDB_SQL_RETRIEVE_TAGS) { + if (pCmd->groupbyExpr.numOfGroupCols > 0) { + setErrMsg(pCmd, msg5); + return TSDB_CODE_INVALID_SQL; + } else { + return TSDB_CODE_SUCCESS; + } + } + + if (pCmd->groupbyExpr.numOfGroupCols > 0) { + SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + int16_t bytes = 0; + int16_t type = 0; + char* name = NULL; + + // check if all the tags prj columns belongs to the group by columns + if (onlyTagPrjFunction(pCmd) && allTagPrjInGroupby(pCmd)) { + updateTagPrjFunction(pCmd); + return doAddGroupbyColumnsOnDemand(pCmd); + } + + // check all query functions in selection clause, multi-output functions are not allowed + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); + int32_t functId = pExpr->functionId; + + /* + * group by normal columns. + * Check if the column projection is identical to the group by column or not + */ + if (functId == TSDB_FUNC_PRJ) { + bool qualified = false; + for (int32_t j = 0; j < pCmd->groupbyExpr.numOfGroupCols; ++j) { + SColIndexEx* pColIndex = &pCmd->groupbyExpr.columnInfo[j]; + if (pColIndex->colId == pExpr->colInfo.colId) { + qualified = true; + break; + } + } + + if (!qualified) { + return TSDB_CODE_INVALID_SQL; + } + } + + if (IS_MULTIOUTPUT(aAggs[functId].nStatus) && functId != TSDB_FUNC_TOP && functId != TSDB_FUNC_BOTTOM && + functId != TSDB_FUNC_TAGPRJ) { + setErrMsg(pCmd, msg1); + return TSDB_CODE_INVALID_SQL; + } + + if (functId == TSDB_FUNC_COUNT && pExpr->colInfo.colIdx == TSDB_TBNAME_COLUMN_INDEX) { + setErrMsg(pCmd, msg1); + return TSDB_CODE_INVALID_SQL; + } + } + + if (checkUpdateTagPrjFunctions(pCmd) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_INVALID_SQL; + } + + /* + * group by tag function must be not changed the function name, otherwise, the group operation may fail to + * divide the subset of final result. + */ + if (doAddGroupbyColumnsOnDemand(pCmd) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_INVALID_SQL; + } + + // projection query on metric does not compatible with "group by" syntax + if (tscProjectionQueryOnMetric(pCmd)) { + setErrMsg(pCmd, msg3); + return TSDB_CODE_INVALID_SQL; + } + } else { + return checkUpdateTagPrjFunctions(pCmd); + } +} diff --git a/src/client/src/tscSQLParserImpl.c b/src/client/src/tscSQLParserImpl.c index f4c581148b..7e1b4a7cf1 100644 --- a/src/client/src/tscSQLParserImpl.c +++ b/src/client/src/tscSQLParserImpl.c @@ -15,13 +15,9 @@ #include #include - #include #include #include - -#include -#include #include #include "os.h" @@ -119,31 +115,29 @@ void tSQLExprListDestroy(tSQLExprList *pList) { free(pList); } -tSQLExpr *tSQLExprIdValueCreate(SSQLToken *pToken, int32_t optrType) { +tSQLExpr *tSQLExprIdValueCreate(SSQLToken *pAliasToken, int32_t optrType) { tSQLExpr *nodePtr = calloc(1, sizeof(tSQLExpr)); if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { - toTSDBType(pToken->type); + toTSDBType(pAliasToken->type); - tVariantCreate(&nodePtr->val, pToken); + tVariantCreate(&nodePtr->val, pAliasToken); nodePtr->nSQLOptr = optrType; } else if (optrType == TK_NOW) { // default use microsecond nodePtr->val.i64Key = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); nodePtr->val.nType = TSDB_DATA_TYPE_BIGINT; - nodePtr->nSQLOptr = TK_TIMESTAMP; - // TK_TIMESTAMP used to denote the time value is in microsecond + nodePtr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond } else if (optrType == TK_VARIABLE) { - int32_t ret = getTimestampInUsFromStr(pToken->z, pToken->n, &nodePtr->val.i64Key); + int32_t ret = getTimestampInUsFromStr(pAliasToken->z, pAliasToken->n, &nodePtr->val.i64Key); UNUSED(ret); nodePtr->val.nType = TSDB_DATA_TYPE_BIGINT; nodePtr->nSQLOptr = TK_TIMESTAMP; - } else { // must be field id if not numbers - assert(optrType == TK_ALL || optrType == TK_ID); - - if (pToken != NULL) { // it must be the column name (tk_id) - nodePtr->colInfo = *pToken; + } else { // it must be the column name (tk_id) if it is not the number + assert(optrType == TK_ID || optrType == TK_ALL); + if (pAliasToken != NULL) { + nodePtr->colInfo = *pAliasToken; } nodePtr->nSQLOptr = optrType; @@ -273,11 +267,10 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { return pExpr; } -void tSQLExprDestroy(tSQLExpr *pExpr) { - if (pExpr == NULL) return; - - tSQLExprDestroy(pExpr->pLeft); - tSQLExprDestroy(pExpr->pRight); +void tSQLExprNodeDestroy(tSQLExpr *pExpr) { + if (pExpr == NULL) { + return; + } if (pExpr->nSQLOptr == TK_STRING) { tVariantDestroy(&pExpr->val); @@ -288,6 +281,17 @@ void tSQLExprDestroy(tSQLExpr *pExpr) { free(pExpr); } +void tSQLExprDestroy(tSQLExpr *pExpr) { + if (pExpr == NULL) { + return; + } + + tSQLExprDestroy(pExpr->pLeft); + tSQLExprDestroy(pExpr->pRight); + + tSQLExprNodeDestroy(pExpr); +} + static void *tVariantListExpand(tVariantList *pList) { if (pList->nAlloc <= pList->nExpr) { // int32_t newSize = (pList->nAlloc << 1) + 4; @@ -420,6 +424,44 @@ void setDBName(SSQLToken *pCpxName, SSQLToken *pDB) { pCpxName->n = pDB->n; } +int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t *result) { + *result = val; + + switch (unit) { + case 's': + (*result) *= MILLISECOND_PER_SECOND; + break; + case 'm': + (*result) *= MILLISECOND_PER_MINUTE; + break; + case 'h': + (*result) *= MILLISECOND_PER_HOUR; + break; + case 'd': + (*result) *= MILLISECOND_PER_DAY; + break; + case 'w': + (*result) *= MILLISECOND_PER_WEEK; + break; + case 'n': + (*result) *= MILLISECOND_PER_MONTH; + break; + case 'y': + (*result) *= MILLISECOND_PER_YEAR; + break; + case 'a': + break; + default: { + ; + return -1; + } + } + + /* get the value in microsecond */ + (*result) *= 1000L; + return 0; +} + void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType) { int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); /* truncate the column name */ @@ -462,7 +504,7 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) { /* * extract the select info out of sql string */ -SQuerySQL *tSetQuerySQLElems(SSQLToken *pSelectToken, tSQLExprList *pSelection, SSQLToken *pFrom, tSQLExpr *pWhere, +SQuerySQL *tSetQuerySQLElems(SSQLToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, tVariantList *pGroupby, tVariantList *pSortOrder, SSQLToken *pInterval, SSQLToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) { assert(pSelection != NULL && pFrom != NULL && pInterval != NULL && pLimit != NULL && pGLimit != NULL); @@ -472,13 +514,13 @@ SQuerySQL *tSetQuerySQLElems(SSQLToken *pSelectToken, tSQLExprList *pSelection, pQuery->selectToken.n = strlen(pQuery->selectToken.z); // all later sql string are belonged to the stream sql pQuery->pSelection = pSelection; - pQuery->from = *pFrom; + pQuery->from = pFrom; pQuery->pGroupby = pGroupby; pQuery->pSortOrder = pSortOrder; pQuery->pWhere = pWhere; pQuery->limit = *pLimit; - pQuery->glimit = *pGLimit; + pQuery->slimit = *pGLimit; pQuery->interval = *pInterval; pQuery->sliding = *pSliding; @@ -532,6 +574,9 @@ void destroyQuerySql(SQuerySQL *pSql) { tVariantListDestroy(pSql->pGroupby); pSql->pGroupby = NULL; + tVariantListDestroy(pSql->from); + pSql->from = NULL; + tVariantListDestroy(pSql->fillType); free(pSql); diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 85e4b3bbef..9728811ae7 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -20,6 +20,7 @@ #include "os.h" #include "taosmsg.h" #include "tschemautil.h" +#include "tsqldef.h" #include "ttypes.h" #include "tutil.h" @@ -71,7 +72,7 @@ struct SSchema* tsGetSchema(SMeterMeta* pMeta) { if (pMeta == NULL) { return NULL; } - return tsGetSchemaColIdx(pMeta, 0); + return tsGetColumnSchema(pMeta, 0); } struct SSchema* tsGetTagSchema(SMeterMeta* pMeta) { @@ -79,24 +80,32 @@ struct SSchema* tsGetTagSchema(SMeterMeta* pMeta) { return NULL; } - return tsGetSchemaColIdx(pMeta, pMeta->numOfColumns); + return tsGetColumnSchema(pMeta, pMeta->numOfColumns); } -struct SSchema* tsGetSchemaColIdx(SMeterMeta* pMeta, int32_t startCol) { - if (pMeta->pSchema == 0) { - pMeta->pSchema = sizeof(SMeterMeta); - } - - return (SSchema*)(((char*)pMeta + pMeta->pSchema) + startCol * sizeof(SSchema)); +struct SSchema* tsGetColumnSchema(SMeterMeta* pMeta, int32_t startCol) { + return (SSchema*)(((char*)pMeta + sizeof(SMeterMeta)) + startCol * sizeof(SSchema)); } +/** + * the MeterMeta data format in memory is as follows: + * + * +--------------------+ + * |SMeterMeta Body data| sizeof(SMeterMeta) + * +--------------------+ + * |Schema data | numOfTotalColumns * sizeof(SSchema) + * +--------------------+ + * |Tags data | tag_col_1.bytes + tag_col_2.bytes + .... + * +--------------------+ + * + * @param pMeta + * @return + */ char* tsGetTagsValue(SMeterMeta* pMeta) { - if (pMeta->tags == 0) { - int32_t numOfTotalCols = pMeta->numOfColumns + pMeta->numOfTags; - pMeta->tags = sizeof(SMeterMeta) + numOfTotalCols * sizeof(SSchema); - } + int32_t numOfTotalCols = pMeta->numOfColumns + pMeta->numOfTags; + uint32_t offset = sizeof(SMeterMeta) + numOfTotalCols * sizeof(SSchema); - return ((char*)pMeta + pMeta->tags); + return ((char*)pMeta + offset); } bool tsMeterMetaIdentical(SMeterMeta* p1, SMeterMeta* p2) { @@ -111,7 +120,7 @@ bool tsMeterMetaIdentical(SMeterMeta* p1, SMeterMeta* p2) { size_t size = sizeof(SMeterMeta) + p1->numOfColumns * sizeof(SSchema); for (int32_t i = 0; i < p1->numOfTags; ++i) { - SSchema* pColSchema = tsGetSchemaColIdx(p1, i + p1->numOfColumns); + SSchema* pColSchema = tsGetColumnSchema(p1, i + p1->numOfColumns); size += pColSchema->bytes; } @@ -142,7 +151,33 @@ void extractMeterName(char* meterId, char* name) { copySegment(name, r, TS_PATH_DELIMITER[0]); } -void extractDBName(char* meterId, char* name) { +SSQLToken extractDBName(char* meterId, char* name) { char* r = skipSegments(meterId, TS_PATH_DELIMITER[0], 1); copySegment(name, r, TS_PATH_DELIMITER[0]); + + SSQLToken token = {.z = name, .n = strlen(name), .type = TK_STRING}; + return token; +} + +/* + * tablePrefix.columnName + * extract table name and save it in pTable, with only column name in pToken + */ +void extractTableNameFromToken(SSQLToken* pToken, SSQLToken* pTable) { + const char sep = TS_PATH_DELIMITER[0]; + + if (pToken == pTable || pToken == NULL || pTable == NULL) { + return; + } + + char* r = strnchr(pToken->z, sep, pToken->n, false); + + if (r != NULL) { // record the table name token + pTable->n = r - pToken->z; + pTable->z = pToken->z; + + r += 1; + pToken->n -= (r - pToken->z); + pToken->z = r; + } } diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index 0683df94ca..ac2638f635 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "tlosertree.h" #include "tlosertree.h" @@ -61,16 +62,17 @@ int32_t treeComparator(const void *pLeft, const void *pRight, void *param) { static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pReducer, tOrderDescriptor *pDesc) { /* * the fields and offset attributes in pCmd and pModel may be different due to - * merge requirement. So, the final result in STscRes structure is formatted in accordance with the pCmd object. + * merge requirement. So, the final result in pRes structure is formatted in accordance with the pCmd object. */ for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { SQLFunctionCtx *pCtx = &pReducer->pCtx[i]; pCtx->aOutputBuf = pReducer->pResultBuf->data + tscFieldInfoGetOffset(pCmd, i) * pReducer->resColModel->maxCapacity; - pCtx->order = pCmd->order.order; - pCtx->aInputElemBuf = - pReducer->pTempBuffer->data + pDesc->pSchema->colOffset[i]; // input buffer hold only one point data + pCtx->functionId = pCmd->exprsInfo.pExprs[i].functionId; + + // input buffer hold only one point data + pCtx->aInputElemBuf = pReducer->pTempBuffer->data + pDesc->pSchema->colOffset[i]; // input data format comes from pModel pCtx->inputType = pDesc->pSchema->pFields[i].type; @@ -80,28 +82,54 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pRedu // output data format yet comes from pCmd. pCtx->outputBytes = pField->bytes; pCtx->outputType = pField->type; - pCtx->numOfOutputElems = 0; - pCtx->numOfIteratedElems = 0; pCtx->startOffset = 0; pCtx->size = 1; - pCtx->hasNullValue = true; + pCtx->hasNull = true; pCtx->currentStage = SECONDARY_STAGE_MERGE; pRes->bytes[i] = pField->bytes; - int32_t sqlFunction = tscSqlExprGet(pCmd, i)->sqlFuncId; - if (sqlFunction == TSDB_FUNC_TOP_DST || sqlFunction == TSDB_FUNC_BOTTOM_DST) { - /* for top_dst/bottom_dst function, the output of timestamp is the first column */ - pCtx->ptsOutputBuf = pReducer->pCtx[0].aOutputBuf; + SSqlExpr *pExpr = tscSqlExprGet(pCmd, i); + // for top/bottom function, the output of timestamp is the first column + int32_t functionId = pExpr->functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + pCtx->ptsOutputBuf = pReducer->pCtx[0].aOutputBuf; pCtx->param[2].i64Key = pCmd->order.order; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->param[3].i64Key = sqlFunction; - pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->param[1].i64Key = pCmd->order.orderColId; } + + SResultInfo *pResInfo = &pReducer->pResInfo[i]; + pResInfo->bufLen = pExpr->interResBytes; + pResInfo->interResultBuf = calloc(1, (size_t)pResInfo->bufLen); + + pCtx->resultInfo = &pReducer->pResInfo[i]; + pCtx->resultInfo->superTableQ = true; + } + + int16_t n = 0; + int16_t tagLen = 0; + SQLFunctionCtx** pTagCtx = calloc(pCmd->fieldsInfo.numOfOutputCols, POINTER_BYTES); + + SQLFunctionCtx* pCtx = NULL; + for(int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + SSqlExpr *pExpr = tscSqlExprGet(pCmd, i); + if (pExpr->functionId == TSDB_FUNC_TAG_DUMMY || pExpr->functionId == TSDB_FUNC_TS_DUMMY) { + tagLen += pExpr->resBytes; + pTagCtx[n++] = &pReducer->pCtx[i]; + } else if ((aAggs[pExpr->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + pCtx = &pReducer->pCtx[i]; + } + } + + if (n == 0) { + free(pTagCtx); + } else { + pCtx->tagInfo.pTagCtxList = pTagCtx; + pCtx->tagInfo.numOfTagCols = n; + pCtx->tagInfo.tagsLen = tagLen; } } @@ -240,13 +268,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->hasUnprocessedRow = false; pReducer->prevRowOfInput = (char *)calloc(1, pReducer->rowSize); - if (pReducer->prevRowOfInput == 0) { - // todo release previously allocated memory - pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; - return; - } - /* used to keep the latest input row */ + // used to keep the latest input row pReducer->pTempBuffer = (tFilePage *)calloc(1, pReducer->rowSize + sizeof(tFilePage)); pReducer->discardData = (tFilePage *)calloc(1, pReducer->rowSize + sizeof(tFilePage)); @@ -254,44 +277,56 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->nResultBufSize = pMemBuffer[0]->nPageSize * 16; pReducer->pResultBuf = (tFilePage *)calloc(1, pReducer->nResultBufSize + sizeof(tFilePage)); - + int32_t finalRowLength = tscGetResRowLength(pCmd); pReducer->resColModel = finalmodel; pReducer->resColModel->maxCapacity = pReducer->nResultBufSize / finalRowLength; assert(finalRowLength <= pReducer->rowSize); + pReducer->pFinalRes = calloc(1, pReducer->rowSize * pReducer->resColModel->maxCapacity); pReducer->pBufForInterpo = calloc(1, pReducer->nResultBufSize); - if (pReducer->pTempBuffer == NULL || pReducer->pResultBuf == NULL || pReducer->pBufForInterpo == NULL) { + if (pReducer->pTempBuffer == NULL|| pReducer->discardData == NULL || pReducer->pResultBuf == NULL || + pReducer->pBufForInterpo == NULL || pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) { + tfree(pReducer->pTempBuffer); + tfree(pReducer->discardData); + tfree(pReducer->pResultBuf); + tfree(pReducer->pFinalRes); + tfree(pReducer->pBufForInterpo); + tfree(pReducer->prevRowOfInput); + pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; return; } pReducer->pTempBuffer->numOfElems = 0; + pReducer->pResInfo = calloc((size_t)pCmd->fieldsInfo.numOfOutputCols, sizeof(SResultInfo)); tscCreateResPointerInfo(pCmd, pRes); tscInitSqlContext(pCmd, pRes, pReducer, pDesc); - /* we change the maxCapacity of schema to denote that there is only one row in temp buffer */ + // we change the maxCapacity of schema to denote that there is only one row in temp buffer pReducer->pDesc->pSchema->maxCapacity = 1; pReducer->offset = pCmd->limit.offset; pRes->pLocalReducer = pReducer; pRes->numOfGroups = 0; - int16_t prec = pCmd->pMeterMeta->precision; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + int16_t prec = pMeterMetaInfo->pMeterMeta->precision; + int64_t stime = (pCmd->stime < pCmd->etime) ? pCmd->stime : pCmd->etime; int64_t revisedSTime = taosGetIntervalStartTimestamp(stime, pCmd->nAggTimeInterval, pCmd->intervalTimeUnit, prec); SInterpolationInfo *pInterpoInfo = &pReducer->interpolationInfo; - taosInitInterpoInfo(pInterpoInfo, pCmd->order.order, revisedSTime, pCmd->groupbyExpr.numOfGroupbyCols, + taosInitInterpoInfo(pInterpoInfo, pCmd->order.order, revisedSTime, pCmd->groupbyExpr.numOfGroupCols, pReducer->rowSize); - int32_t startIndex = pCmd->fieldsInfo.numOfOutputCols - pCmd->groupbyExpr.numOfGroupbyCols; + int32_t startIndex = pCmd->fieldsInfo.numOfOutputCols - pCmd->groupbyExpr.numOfGroupCols; - if (pCmd->groupbyExpr.numOfGroupbyCols > 0) { - pInterpoInfo->pTags[0] = (char *)pInterpoInfo->pTags + POINTER_BYTES * pCmd->groupbyExpr.numOfGroupbyCols; - for (int32_t i = 1; i < pCmd->groupbyExpr.numOfGroupbyCols; ++i) { + if (pCmd->groupbyExpr.numOfGroupCols > 0) { + pInterpoInfo->pTags[0] = (char *)pInterpoInfo->pTags + POINTER_BYTES * pCmd->groupbyExpr.numOfGroupCols; + for (int32_t i = 1; i < pCmd->groupbyExpr.numOfGroupCols; ++i) { pInterpoInfo->pTags[i] = pReducer->resColModel->pFields[startIndex + i - 1].bytes + pInterpoInfo->pTags[i - 1]; } } else { @@ -399,6 +434,8 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { return; } + SSqlCmd *pCmd = &pSql->cmd; + // there is no more result, so we release all allocated resource SLocalReducer *pLocalReducer = (SLocalReducer *)__sync_val_compare_and_swap_64(&pRes->pLocalReducer, pRes->pLocalReducer, 0); @@ -413,12 +450,29 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { tfree(pLocalReducer->interpolationInfo.prevValues); tfree(pLocalReducer->interpolationInfo.pTags); - tfree(pLocalReducer->pCtx); + if (pLocalReducer->pCtx != NULL) { + for(int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i]; + tVariantDestroy(&pCtx->tag); + } + + tfree(pLocalReducer->pCtx); + } + + tfree(pLocalReducer->prevRowOfInput); tfree(pLocalReducer->pTempBuffer); tfree(pLocalReducer->pResultBuf); + if (pLocalReducer->pResInfo != NULL) { + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + tfree(pLocalReducer->pResInfo[i].interResultBuf); + } + + tfree(pLocalReducer->pResInfo); + } + if (pLocalReducer->pLoserTree) { tfree(pLocalReducer->pLoserTree->param); tfree(pLocalReducer->pLoserTree); @@ -447,8 +501,8 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCmd, tColModel *pModel) { int32_t numOfGroupByCols = 0; - if (pCmd->groupbyExpr.numOfGroupbyCols > 0) { - numOfGroupByCols = pCmd->groupbyExpr.numOfGroupbyCols; + if (pCmd->groupbyExpr.numOfGroupCols > 0) { + numOfGroupByCols = pCmd->groupbyExpr.numOfGroupCols; } // primary timestamp column is involved in final result @@ -462,15 +516,15 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm } if (numOfGroupByCols > 0) { - int32_t startCols = pCmd->fieldsInfo.numOfOutputCols - pCmd->groupbyExpr.numOfGroupbyCols; + int32_t startCols = pCmd->fieldsInfo.numOfOutputCols - pCmd->groupbyExpr.numOfGroupCols; // tags value locate at the last columns - for (int32_t i = 0; i < pCmd->groupbyExpr.numOfGroupbyCols; ++i) { + for (int32_t i = 0; i < pCmd->groupbyExpr.numOfGroupCols; ++i) { orderIdx[i] = startCols++; } if (pCmd->nAggTimeInterval != 0) { - //the first column is the timestamp, handles queries like "interval(10m) group by tags" + // the first column is the timestamp, handles queries like "interval(10m) group by tags" orderIdx[numOfGroupByCols - 1] = PRIMARYKEY_TIMESTAMP_COL_INDEX; } } @@ -486,7 +540,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm } bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage *tmpBuffer) { - int16_t functionId = tscSqlExprGet(pCmd, 0)->sqlFuncId; + int16_t functionId = tscSqlExprGet(pCmd, 0)->functionId; // disable merge procedure for column projection query if (functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_ARITHM) { @@ -525,7 +579,9 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr tColModel *pModel = NULL; *pFinalModel = NULL; - (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pCmd->pMetricMeta->numOfVnodes); + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + + (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pMeterMetaInfo->pMetricMeta->numOfVnodes); if (*pMemBuffer == NULL) { tscError("%p failed to allocate memory", pSql); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; @@ -552,10 +608,10 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr int32_t capacity = nBufferSizes / rlen; pModel = tColModelCreate(pSchema, pCmd->fieldsInfo.numOfOutputCols, capacity); - for (int32_t i = 0; i < pCmd->pMetricMeta->numOfVnodes; ++i) { + for (int32_t i = 0; i < pMeterMetaInfo->pMetricMeta->numOfVnodes; ++i) { char tmpPath[512] = {0}; - getExtTmpfilePath("/tv_bf_db_%lld_%lld_%d.d", taosGetPthreadId(), i, 0, tmpPath); - tscTrace("%p create tmp file:%s", pSql, tmpPath); + getTmpfilePath("tv_bf_db", tmpPath); + tscTrace("%p create [%d](%d) tmp file for subquery:%s", pSql, pMeterMetaInfo->pMetricMeta->numOfVnodes, i, tmpPath); tExtMemBufferCreate(&(*pMemBuffer)[i], nBufferSizes, rlen, tmpPath, pModel); (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; @@ -667,11 +723,13 @@ void adjustLoserTreeFromNewData(SLocalReducer *pLocalReducer, SLocalDataSource * void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SSqlCmd *pCmd, SInterpolationInfo *pInterpoInfo) { // discard following dataset in the same group and reset the interpolation information - int16_t prec = pCmd->pMeterMeta->precision; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + int16_t prec = pMeterMetaInfo->pMeterMeta->precision; + int64_t stime = (pCmd->stime < pCmd->etime) ? pCmd->stime : pCmd->etime; int64_t revisedSTime = taosGetIntervalStartTimestamp(stime, pCmd->nAggTimeInterval, pCmd->intervalTimeUnit, prec); - taosInitInterpoInfo(pInterpoInfo, pCmd->order.order, revisedSTime, pCmd->groupbyExpr.numOfGroupbyCols, + taosInitInterpoInfo(pInterpoInfo, pCmd->order.order, revisedSTime, pCmd->groupbyExpr.numOfGroupCols, pLocalReducer->rowSize); pLocalReducer->discard = true; @@ -735,10 +793,6 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo assert(pRes->pLocalReducer == NULL); } - if (pLocalReducer->pFinalRes == NULL) { - pLocalReducer->pFinalRes = malloc(pLocalReducer->rowSize * pLocalReducer->resColModel->maxCapacity); - } - if (pCmd->nAggTimeInterval == 0 || pCmd->interpoType == TSDB_INTERPO_NONE) { // no interval query, no interpolation pRes->data = pLocalReducer->pFinalRes; @@ -808,10 +862,11 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { srcData[i] = pLocalReducer->pBufForInterpo + tscFieldInfoGetOffset(pCmd, i) * pInterpoInfo->numOfRawDataInRows; - functions[i] = tscSqlExprGet(pCmd, i)->sqlFuncId; + functions[i] = tscSqlExprGet(pCmd, i)->functionId; } - int8_t precision = pCmd->pMeterMeta->precision; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + int8_t precision = pMeterMetaInfo->pMeterMeta->precision; while (1) { int32_t remains = taosNumOfRemainPoints(pInterpoInfo); @@ -911,22 +966,42 @@ static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) pLocalReducer->hasPrevRow = true; } -static void handleUnprocessedRow(SLocalReducer *pLocalReducer, SSqlCmd *pCmd, tFilePage *tmpBuffer) { - if (pLocalReducer->hasUnprocessedRow) { - for (int32_t j = 0; j < pCmd->fieldsInfo.numOfOutputCols; ++j) { - SSqlExpr *pExpr = tscSqlExprGet(pCmd, j); +static void doExecuteSecondaryMerge(SSqlCmd* pCmd, SLocalReducer *pLocalReducer, bool needInit) { + // the tag columns need to be set before all functions execution + for(int32_t j = 0; j < pCmd->fieldsInfo.numOfOutputCols; ++j) { + SSqlExpr * pExpr = tscSqlExprGet(pCmd, j); + SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[j]; - tVariantAssign(&pLocalReducer->pCtx[j].param[0], &pExpr->param[0]); - aAggs[pExpr->sqlFuncId].init(&pLocalReducer->pCtx[j]); + tVariantAssign(&pCtx->param[0], &pExpr->param[0]); - pLocalReducer->pCtx[j].currentStage = SECONDARY_STAGE_MERGE; - pLocalReducer->pCtx[j].numOfIteratedElems = 0; - aAggs[pExpr->sqlFuncId].distSecondaryMergeFunc(&pLocalReducer->pCtx[j]); + // tags/tags_dummy function, the tag field of SQLFunctionCtx is from the input buffer + if (pExpr->functionId == TSDB_FUNC_TAG_DUMMY || pExpr->functionId == TSDB_FUNC_TAG || + pExpr->functionId == TSDB_FUNC_TS_DUMMY) { + tVariantDestroy(&pCtx->tag); + tVariantCreateFromBinary(&pCtx->tag, pCtx->aInputElemBuf, pCtx->inputBytes, pCtx->inputType); } - pLocalReducer->hasUnprocessedRow = false; + pCtx->currentStage = SECONDARY_STAGE_MERGE; + + if (needInit) { + aAggs[pExpr->functionId].init(pCtx); + } + } + + for (int32_t j = 0; j < pCmd->fieldsInfo.numOfOutputCols; ++j) { + int32_t functionId = tscSqlExprGet(pCmd, j)->functionId; + if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { + continue; + } + + aAggs[functionId].distSecondaryMergeFunc(&pLocalReducer->pCtx[j]); + } +} - // copy to previous temp buffer +static void handleUnprocessedRow(SSqlCmd* pCmd, SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) { + if (pLocalReducer->hasUnprocessedRow) { + pLocalReducer->hasUnprocessedRow = false; + doExecuteSecondaryMerge(pCmd, pLocalReducer, true); savePreviousRow(pLocalReducer, tmpBuffer); } } @@ -935,7 +1010,7 @@ static int64_t getNumOfResultLocal(SSqlCmd *pCmd, SQLFunctionCtx *pCtx) { int64_t maxOutput = 0; for (int32_t j = 0; j < pCmd->exprsInfo.numOfExprs; ++j) { - int32_t functionId = tscSqlExprGet(pCmd, j)->sqlFuncId; + int32_t functionId = tscSqlExprGet(pCmd, j)->functionId; /* * ts, tag, tagprj function can not decide the output number of current query @@ -945,34 +1020,37 @@ static int64_t getNumOfResultLocal(SSqlCmd *pCmd, SQLFunctionCtx *pCtx) { continue; } - if (maxOutput < pCtx[j].numOfOutputElems) { - maxOutput = pCtx[j].numOfOutputElems; + if (maxOutput < GET_RES_INFO(&pCtx[j])->numOfRes) { + maxOutput = GET_RES_INFO(&pCtx[j])->numOfRes; } } return maxOutput; } /* - * in handling the to/bottom query, which produce more than one rows result, + * in handling the top/bottom query, which produce more than one rows result, * the tsdb_func_tags only fill the first row of results, the remain rows need to * filled with the same result, which is the tags, specified in group by clause + * */ static void fillMultiRowsOfTagsVal(SSqlCmd *pCmd, int32_t numOfRes, SLocalReducer *pLocalReducer) { - int32_t startIndex = pCmd->fieldsInfo.numOfOutputCols - pCmd->groupbyExpr.numOfGroupbyCols; - - int32_t maxBufSize = 0; - for (int32_t k = startIndex; k < pCmd->fieldsInfo.numOfOutputCols; ++k) { + int32_t maxBufSize = 0; // find the max tags column length to prepare the buffer + for (int32_t k = 0; k < pCmd->fieldsInfo.numOfOutputCols; ++k) { SSqlExpr *pExpr = tscSqlExprGet(pCmd, k); - if (maxBufSize < pExpr->resBytes) { + if (maxBufSize < pExpr->resBytes && pExpr->functionId == TSDB_FUNC_TAG) { maxBufSize = pExpr->resBytes; } - assert(pExpr->sqlFuncId == TSDB_FUNC_TAG); } assert(maxBufSize >= 0); - char *buf = malloc((size_t)maxBufSize); - for (int32_t k = startIndex; k < pCmd->fieldsInfo.numOfOutputCols; ++k) { + char *buf = malloc((size_t) maxBufSize); + for (int32_t k = 0; k < pCmd->fieldsInfo.numOfOutputCols; ++k) { + SSqlExpr *pExpr = tscSqlExprGet(pCmd, k); + if (pExpr->functionId != TSDB_FUNC_TAG) { + continue; + } + int32_t inc = numOfRes - 1; // tsdb_func_tag function only produce one row of result memset(buf, 0, (size_t)maxBufSize); @@ -991,7 +1069,10 @@ static void fillMultiRowsOfTagsVal(SSqlCmd *pCmd, int32_t numOfRes, SLocalReduce int32_t finalizeRes(SSqlCmd *pCmd, SLocalReducer *pLocalReducer) { for (int32_t k = 0; k < pCmd->fieldsInfo.numOfOutputCols; ++k) { SSqlExpr *pExpr = tscSqlExprGet(pCmd, k); - aAggs[pExpr->sqlFuncId].xFinalize(&pLocalReducer->pCtx[k]); + aAggs[pExpr->functionId].xFinalize(&pLocalReducer->pCtx[k]); + + // allow to re-initialize for the next round + pLocalReducer->pCtx[k].resultInfo->initialized = false; } pLocalReducer->hasPrevRow = false; @@ -1012,7 +1093,7 @@ int32_t finalizeRes(SSqlCmd *pCmd, SLocalReducer *pLocalReducer) { */ bool needToMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) { int32_t ret = 0; // merge all result by default - int16_t functionId = tscSqlExprGet(pCmd, 0)->sqlFuncId; + int16_t functionId = tscSqlExprGet(pCmd, 0)->functionId; if (functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_ARITHM) { // column projection query ret = 1; // disable merge procedure @@ -1033,7 +1114,7 @@ bool needToMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, tFilePage *tmpBuff } static bool reachGroupResultLimit(SSqlCmd *pCmd, SSqlRes *pRes) { - return (pRes->numOfGroups >= pCmd->glimit.limit && pCmd->glimit.limit >= 0); + return (pRes->numOfGroups >= pCmd->slimit.limit && pCmd->slimit.limit >= 0); } static bool saveGroupResultInfo(SSqlObj *pSql) { @@ -1042,7 +1123,7 @@ static bool saveGroupResultInfo(SSqlObj *pSql) { pRes->numOfGroups += 1; - // the output group is limited by the glimit clause + // the output group is limited by the slimit clause if (reachGroupResultLimit(pCmd, pRes)) { return true; } @@ -1073,9 +1154,9 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no * ignore the output of the current group since this group is skipped by user * We set the numOfRows to be 0 and discard the possible remain results. */ - if (pCmd->glimit.offset > 0) { + if (pCmd->slimit.offset > 0) { pRes->numOfRows = 0; - pCmd->glimit.offset -= 1; + pCmd->slimit.offset -= 1; pLocalReducer->discard = !noMoreCurrentGroupRes; return false; } @@ -1089,9 +1170,9 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no #endif SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo; - int32_t startIndex = pCmd->fieldsInfo.numOfOutputCols - pCmd->groupbyExpr.numOfGroupbyCols; + int32_t startIndex = pCmd->fieldsInfo.numOfOutputCols - pCmd->groupbyExpr.numOfGroupCols; - for (int32_t i = 0; i < pCmd->groupbyExpr.numOfGroupbyCols; ++i) { + for (int32_t i = 0; i < pCmd->groupbyExpr.numOfGroupCols; ++i) { memcpy(pInterpoInfo->pTags[i], pLocalReducer->pBufForInterpo + pModel->colOffset[startIndex + i] * pResBuf->numOfElems, pModel->pFields[startIndex + i].bytes); @@ -1113,20 +1194,21 @@ void resetOutputBuf(SSqlCmd *pCmd, SLocalReducer *pLocalReducer) { // reset out } static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer *pLocalReducer) { - //In handling data in other groups, we need to reset the interpolation information for a new group data + // In handling data in other groups, we need to reset the interpolation information for a new group data pRes->numOfRows = 0; pRes->numOfTotal = 0; pCmd->limit.offset = pLocalReducer->offset; - int16_t precision = pCmd->pMeterMeta->precision; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + int16_t precision = pMeterMetaInfo->pMeterMeta->precision; + // for group result interpolation, do not return if not data is generated if (pCmd->interpoType != TSDB_INTERPO_NONE) { - /* for group result interpolation, do not return if not data is generated */ int64_t stime = (pCmd->stime < pCmd->etime) ? pCmd->stime : pCmd->etime; int64_t newTime = taosGetIntervalStartTimestamp(stime, pCmd->nAggTimeInterval, pCmd->intervalTimeUnit, precision); - taosInitInterpoInfo(&pLocalReducer->interpolationInfo, pCmd->order.order, newTime, - pCmd->groupbyExpr.numOfGroupbyCols, pLocalReducer->rowSize); + taosInitInterpoInfo(&pLocalReducer->interpolationInfo, pCmd->order.order, newTime, pCmd->groupbyExpr.numOfGroupCols, + pLocalReducer->rowSize); } } @@ -1140,7 +1222,9 @@ static bool doInterpolationForCurrentGroup(SSqlObj *pSql) { SLocalReducer * pLocalReducer = pRes->pLocalReducer; SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo; - int8_t p = pCmd->pMeterMeta->precision; + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + int8_t p = pMeterMetaInfo->pMeterMeta->precision; if (taosHasRemainsDataForInterpolation(pInterpoInfo)) { assert(pCmd->interpoType != TSDB_INTERPO_NONE); @@ -1170,7 +1254,9 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo; bool prevGroupCompleted = (!pLocalReducer->discard) && pLocalReducer->hasUnprocessedRow; - int8_t precision = pCmd->pMeterMeta->precision; + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + int8_t precision = pMeterMetaInfo->pMeterMeta->precision; if ((isAllSourcesCompleted(pLocalReducer) && !pLocalReducer->hasPrevRow) || pLocalReducer->pLocalDataSrc[0] == NULL || prevGroupCompleted) { @@ -1214,40 +1300,17 @@ static void doMergeWithPrevRows(SSqlObj *pSql, int32_t numOfRes) { for (int32_t k = 0; k < pCmd->fieldsInfo.numOfOutputCols; ++k) { SSqlExpr *pExpr = tscSqlExprGet(pCmd, k); + SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[k]; - pLocalReducer->pCtx[k].aOutputBuf += pLocalReducer->pCtx[k].outputBytes * numOfRes; + pCtx->aOutputBuf += pCtx->outputBytes * numOfRes; // set the correct output timestamp column position - if (pExpr->sqlFuncId == TSDB_FUNC_TOP_DST || pExpr->sqlFuncId == TSDB_FUNC_BOTTOM_DST) { - pLocalReducer->pCtx[k].ptsOutputBuf = ((char *)pLocalReducer->pCtx[k].ptsOutputBuf + TSDB_KEYSIZE * numOfRes); + if (pExpr->functionId == TSDB_FUNC_TOP || pExpr->functionId == TSDB_FUNC_BOTTOM) { + pCtx->ptsOutputBuf = ((char *)pCtx->ptsOutputBuf + TSDB_KEYSIZE * numOfRes); } - - /* set the parameters for the SQLFunctionCtx */ - tVariantAssign(&pLocalReducer->pCtx[k].param[0], &pExpr->param[0]); - - aAggs[pExpr->sqlFuncId].init(&pLocalReducer->pCtx[k]); - pLocalReducer->pCtx[k].currentStage = SECONDARY_STAGE_MERGE; - aAggs[pExpr->sqlFuncId].distSecondaryMergeFunc(&pLocalReducer->pCtx[k]); } -} -static void doExecuteSecondaryMerge(SSqlObj *pSql) { - SSqlCmd * pCmd = &pSql->cmd; - SSqlRes * pRes = &pSql->res; - SLocalReducer *pLocalReducer = pRes->pLocalReducer; - - for (int32_t j = 0; j < pCmd->fieldsInfo.numOfOutputCols; ++j) { - SSqlExpr *pExpr = tscSqlExprGet(pCmd, j); - - tVariantAssign(&pLocalReducer->pCtx[j].param[0], &pExpr->param[0]); - pLocalReducer->pCtx[j].numOfIteratedElems = 0; - pLocalReducer->pCtx[j].currentStage = 0; - - aAggs[pExpr->sqlFuncId].init(&pLocalReducer->pCtx[j]); - pLocalReducer->pCtx[j].currentStage = SECONDARY_STAGE_MERGE; - - aAggs[pExpr->sqlFuncId].distSecondaryMergeFunc(&pLocalReducer->pCtx[j]); - } + doExecuteSecondaryMerge(pCmd, pLocalReducer, true); } int32_t tscLocalDoReduce(SSqlObj *pSql) { @@ -1272,8 +1335,7 @@ int32_t tscLocalDoReduce(SSqlObj *pSql) { int32_t prevStatus = __sync_val_compare_and_swap_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY, TSC_LOCALREDUCE_IN_PROGRESS); if (prevStatus != TSC_LOCALREDUCE_READY || pLocalReducer == NULL) { - assert(prevStatus == TSC_LOCALREDUCE_TOBE_FREED); - /* it is in tscDestroyLocalReducer function already */ + assert(prevStatus == TSC_LOCALREDUCE_TOBE_FREED); // it is in tscDestroyLocalReducer function already return TSDB_CODE_SUCCESS; } @@ -1292,7 +1354,7 @@ int32_t tscLocalDoReduce(SSqlObj *pSql) { SLoserTreeInfo *pTree = pLocalReducer->pLoserTree; // clear buffer - handleUnprocessedRow(pLocalReducer, pCmd, tmpBuffer); + handleUnprocessedRow(pCmd, pLocalReducer, tmpBuffer); tColModel *pModel = pLocalReducer->pDesc->pSchema; while (1) { @@ -1353,12 +1415,7 @@ int32_t tscLocalDoReduce(SSqlObj *pSql) { if (pLocalReducer->hasPrevRow) { if (needToMerge(pCmd, pLocalReducer, tmpBuffer)) { // belong to the group of the previous row, continue process it - for (int32_t j = 0; j < pCmd->fieldsInfo.numOfOutputCols; ++j) { - SSqlExpr *pExpr = tscSqlExprGet(pCmd, j); - tVariantAssign(&pLocalReducer->pCtx[j].param[0], &pExpr->param[0]); - - aAggs[pExpr->sqlFuncId].distSecondaryMergeFunc(&pLocalReducer->pCtx[j]); - } + doExecuteSecondaryMerge(pCmd, pLocalReducer, false); // copy to buffer savePreviousRow(pLocalReducer, tmpBuffer); @@ -1369,7 +1426,7 @@ int32_t tscLocalDoReduce(SSqlObj *pSql) { */ int32_t numOfRes = finalizeRes(pCmd, pLocalReducer); - bool sameGroup = isSameGroup(pCmd, pLocalReducer, pLocalReducer->prevRowOfInput, tmpBuffer); + bool sameGroup = isSameGroup(pCmd, pLocalReducer, pLocalReducer->prevRowOfInput, tmpBuffer); tFilePage *pResBuf = pLocalReducer->pResultBuf; /* @@ -1398,7 +1455,7 @@ int32_t tscLocalDoReduce(SSqlObj *pSql) { assert(pLocalReducer->status == TSC_LOCALREDUCE_IN_PROGRESS); if (pRes->numOfRows == 0) { - handleUnprocessedRow(pLocalReducer, pCmd, tmpBuffer); + handleUnprocessedRow(pCmd, pLocalReducer, tmpBuffer); if (!sameGroup) { /* @@ -1418,7 +1475,7 @@ int32_t tscLocalDoReduce(SSqlObj *pSql) { * We start the process in a new round. */ if (sameGroup) { - handleUnprocessedRow(pLocalReducer, pCmd, tmpBuffer); + handleUnprocessedRow(pCmd, pLocalReducer, tmpBuffer); } } @@ -1435,7 +1492,7 @@ int32_t tscLocalDoReduce(SSqlObj *pSql) { } } } else { - doExecuteSecondaryMerge(pSql); + doExecuteSecondaryMerge(pCmd, pLocalReducer, true); savePreviousRow(pLocalReducer, tmpBuffer); // copy the processed row to buffer } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index ec48430453..6edec3d477 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -14,6 +14,7 @@ */ #include +#include #include #include #include @@ -21,11 +22,13 @@ #include "os.h" #include "tcache.h" #include "trpc.h" +#include "tscJoinProcess.h" #include "tscProfile.h" #include "tscSecondaryMerge.h" #include "tscUtil.h" #include "tschemautil.h" #include "tsclient.h" +#include "tscompression.h" #include "tsocket.h" #include "tsql.h" #include "ttime.h" @@ -34,18 +37,34 @@ #define TSC_MGMT_VNODE 999 -int tsMasterIndex = 0; -int tsSlaveIndex = 0; // slave == master for single node edition -uint32_t tsServerIp; +#ifdef CLUSTER + SIpStrList tscMgmtIpList; + int tsMasterIndex = 0; + int tsSlaveIndex = 1; +#else + int tsMasterIndex = 0; + int tsSlaveIndex = 0; // slave == master for single node edition + uint32_t tsServerIp; +#endif int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql); int (*tscProcessMsgRsp[TSDB_SQL_MAX])(SSqlObj *pSql); -void (*tscUpdateVnodeMsg[TSDB_SQL_MAX])(SSqlObj *pSql, char* buf); +void (*tscUpdateVnodeMsg[TSDB_SQL_MAX])(SSqlObj *pSql, char *buf); void tscProcessActivityTimer(void *handle, void *tmrId); int tscKeepConn[TSDB_SQL_MAX] = {0}; static int32_t minMsgSize() { return tsRpcHeadSize + sizeof(STaosDigest); } +#ifdef CLUSTER +void tscPrintMgmtIp() { + if (tscMgmtIpList.numOfIps <= 0) { + tscError("invalid IP list:%d", tscMgmtIpList.numOfIps); + } else { + for (int i = 0; i < tscMgmtIpList.numOfIps; ++i) tscTrace("mgmt index:%d ip:%s", i, tscMgmtIpList.ipstr[i]); + } +} +#endif + void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { STscObj *pObj = (STscObj *)param; if (pObj == NULL) return; @@ -59,6 +78,18 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (code == 0) { SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp; +#ifdef CLUSTER + SIpList * pIpList = &pRsp->ipList; + tscMgmtIpList.numOfIps = pIpList->numOfIps; + if (memcmp(tscMgmtIpList.ip, pIpList->ip, pIpList->numOfIps * 4) != 0) { + for (int i = 0; i < pIpList->numOfIps; ++i) { + tinet_ntoa(tscMgmtIpList.ipstr[i], pIpList->ip[i]); + tscMgmtIpList.ip[i] = pIpList->ip[i]; + } + tscTrace("new mgmt IP list:"); + tscPrintMgmtIp(); + } +#endif if (pRsp->killConnection) { tscKillConnection(pObj); } else { @@ -80,16 +111,21 @@ void tscProcessActivityTimer(void *handle, void *tmrId) { if (pObj->pTimer != tmrId) return; if (pObj->pHb == NULL) { - SSqlObj *pSql = (SSqlObj *)malloc(sizeof(SSqlObj)); - memset(pSql, 0, sizeof(SSqlObj)); + SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); + if (NULL == pSql) return; + pSql->fp = tscProcessHeartBeatRsp; pSql->cmd.command = TSDB_SQL_HB; - tscAllocPayloadWithSize(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE); + if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) { + tfree(pSql); + return; + } + pSql->param = pObj; pSql->pTscObj = pObj; pSql->signature = pSql; pObj->pHb = pSql; - tscTrace("%p pHb is allocated, pObj:%p, pSql:%p", pObj->pHb, pObj, pObj->pSql); + tscTrace("%p pHb is allocated, pObj:%p", pObj->pHb, pObj); } if (tscShouldFreeHeatBeat(pObj->pHb)) { @@ -106,11 +142,19 @@ void tscProcessActivityTimer(void *handle, void *tmrId) { void tscGetConnToMgmt(SSqlObj *pSql, uint8_t *pCode) { STscObj *pTscObj = pSql->pTscObj; - +#ifdef CLUSTER + if (pSql->retry < tscMgmtIpList.numOfIps) { + *pCode = 0; + pSql->retry++; + pSql->index = pSql->index % tscMgmtIpList.numOfIps; + if (pSql->cmd.command > TSDB_SQL_READ && pSql->index == 0) pSql->index = 1; + void *thandle = taosGetConnFromCache(tscConnCache, tscMgmtIpList.ip[pSql->index], TSC_MGMT_VNODE, pTscObj->user); +#else if (pSql->retry < 1) { *pCode = 0; pSql->retry++; void *thandle = taosGetConnFromCache(tscConnCache, tsServerIp, TSC_MGMT_VNODE, pTscObj->user); +#endif if (thandle == NULL) { SRpcConnInit connInit; @@ -125,33 +169,45 @@ void tscGetConnToMgmt(SSqlObj *pSql, uint8_t *pCode) { connInit.spi = 1; connInit.encrypt = 0; connInit.secret = pSql->pTscObj->pass; - - connInit.peerIp = tsServerIpStr; +#ifdef CLUSTER + connInit.peerIp = tscMgmtIpList.ipstr[pSql->index]; +#else + connInit.peerIp = tsServerIpStr; +#endif thandle = taosOpenRpcConn(&connInit, pCode); } pSql->thandle = thandle; +#ifdef CLUSTER + pSql->ip = tscMgmtIpList.ip[pSql->index]; + pSql->vnode = TSC_MGMT_VNODE; + tscTrace("%p mgmt index:%d ip:0x%x is picked up, pConn:%p", pSql, pSql->index, tscMgmtIpList.ip[pSql->index], + pSql->thandle); +#else pSql->ip = tsServerIp; pSql->vnode = TSC_MGMT_VNODE; +#endif } } void tscGetConnToVnode(SSqlObj *pSql, uint8_t *pCode) { + char ipstr[40] = {0}; SVPeerDesc *pVPeersDesc = NULL; static int vidIndex = 0; STscObj * pTscObj = pSql->pTscObj; pSql->thandle = NULL; - SSqlCmd *pCmd = &pSql->cmd; - if (UTIL_METER_IS_METRIC(pCmd)) { // multiple vnode query - int32_t idx = (pCmd->vnodeIdx > 0) ? pCmd->vnodeIdx - 1 : 0; - SVnodeSidList *vnodeList = tscGetVnodeSidList(pCmd->pMetricMeta, idx); + SSqlCmd * pCmd = &pSql->cmd; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + + if (UTIL_METER_IS_METRIC(pMeterMetaInfo)) { // multiple vnode query + SVnodeSidList *vnodeList = tscGetVnodeSidList(pMeterMetaInfo->pMetricMeta, pCmd->vnodeIdx); if (vnodeList != NULL) { pVPeersDesc = vnodeList->vpeerDesc; } } else { - SMeterMeta *pMeta = pSql->cmd.pMeterMeta; + SMeterMeta *pMeta = pMeterMetaInfo->pMeterMeta; if (pMeta == NULL) { tscError("%p pMeterMeta is NULL", pSql); pSql->retry = pSql->maxRetry; @@ -167,7 +223,40 @@ void tscGetConnToVnode(SSqlObj *pSql, uint8_t *pCode) { while (pSql->retry < pSql->maxRetry) { (pSql->retry)++; +#ifdef CLUSTER + if (pVPeersDesc[pSql->index].ip == 0) { + (pSql->index) = (pSql->index + 1) % TSDB_VNODES_SUPPORT; + continue; + } + *pCode = 0; + + void *thandle = + taosGetConnFromCache(tscConnCache, pVPeersDesc[pSql->index].ip, pVPeersDesc[pSql->index].vnode, pTscObj->user); + if (thandle == NULL) { + SRpcConnInit connInit; + tinet_ntoa(ipstr, pVPeersDesc[pSql->index].ip); + memset(&connInit, 0, sizeof(connInit)); + connInit.cid = vidIndex; + connInit.sid = 0; + connInit.spi = 0; + connInit.encrypt = 0; + connInit.meterId = pSql->pTscObj->user; + connInit.peerId = htonl((pVPeersDesc[pSql->index].vnode << TSDB_SHELL_VNODE_BITS)); + connInit.shandle = pVnodeConn; + connInit.ahandle = pSql; + connInit.peerIp = ipstr; + connInit.peerPort = tsVnodeShellPort; + thandle = taosOpenRpcConn(&connInit, pCode); + vidIndex = (vidIndex + 1) % tscNumOfThreads; + } + + pSql->thandle = thandle; + pSql->ip = pVPeersDesc[pSql->index].ip; + pSql->vnode = pVPeersDesc[pSql->index].vnode; + tscTrace("%p vnode:%d ip:0x%x index:%d is picked up, pConn:%p", pSql, pVPeersDesc[pSql->index].vnode, + pVPeersDesc[pSql->index].ip, pSql->index, pSql->thandle); +#else *pCode = 0; void *thandle = taosGetConnFromCache(tscConnCache, tsServerIp, pVPeersDesc[0].vnode, pTscObj->user); @@ -191,6 +280,8 @@ void tscGetConnToVnode(SSqlObj *pSql, uint8_t *pCode) { pSql->thandle = thandle; pSql->ip = tsServerIp; pSql->vnode = pVPeersDesc[0].vnode; +#endif + break; } } @@ -214,17 +305,18 @@ int tscSendMsgToServer(SSqlObj *pSql) { * message body by using "if (pHeader->msgType & 1)" may cause the segment fault. * */ - int32_t totalMsgLen = pSql->cmd.payloadLen + tsRpcHeadSize + sizeof(STaosDigest); + size_t totalLen = pSql->cmd.payloadLen + tsRpcHeadSize + sizeof(STaosDigest); // the memory will be released by taosProcessResponse, so no memory leak here - char *buf = malloc(totalMsgLen); + char *buf = malloc(totalLen); if (NULL == buf) { tscError("%p msg:%s malloc fail", pSql, taosMsg[pSql->cmd.msgType]); return TSDB_CODE_CLI_OUT_OF_MEMORY; } + memcpy(buf, pSql->cmd.payload, totalLen); - memcpy(buf, pSql->cmd.payload, (size_t) totalMsgLen); tscTrace("%p msg:%s is sent to server", pSql, taosMsg[pSql->cmd.msgType]); + char *pStart = taosBuildReqHeader(pSql->thandle, pSql->cmd.msgType, buf); if (pStart) { if (tscUpdateVnodeMsg[pSql->cmd.command]) (*tscUpdateVnodeMsg[pSql->cmd.command])(pSql, buf); @@ -238,6 +330,27 @@ int tscSendMsgToServer(SSqlObj *pSql) { return code; } +#ifdef CLUSTER +void tscProcessMgmtRedirect(SSqlObj *pSql, uint8_t *cont) { + SIpList *pIpList = (SIpList *)(cont); + tscMgmtIpList.numOfIps = pIpList->numOfIps; + for (int i = 0; i < pIpList->numOfIps; ++i) { + tinet_ntoa(tscMgmtIpList.ipstr[i], pIpList->ip[i]); + tscMgmtIpList.ip[i] = pIpList->ip[i]; + tscTrace("Update mgmt Ip, index:%d ip:%s", i, tscMgmtIpList.ipstr[i]); + } + + if (pSql->cmd.command < TSDB_SQL_READ) { + tsMasterIndex = 0; + pSql->index = 0; + } else { + pSql->index++; + } + + tscPrintMgmtIp(); +} +#endif + void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { if (ahandle == NULL) return NULL; @@ -261,17 +374,22 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { tscTrace("%p msg:%p is received from server, pConn:%p", pSql, msg, thandle); if (pSql->freed || pObj->signature != pObj) { - tscTrace("%p sql is already released or DB connection is closed, freed:%d pObj:%p signature:%p", - pSql, pSql->freed, pObj, pObj->signature); + tscTrace("%p sql is already released or DB connection is closed, freed:%d pObj:%p signature:%p", pSql, pSql->freed, + pObj, pObj->signature); taosAddConnIntoCache(tscConnCache, pSql->thandle, pSql->ip, pSql->vnode, pObj->user); tscFreeSqlObj(pSql); return ahandle; } + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); if (msg == NULL) { tscTrace("%p no response from ip:0x%x", pSql, pSql->ip); - + +#ifdef CLUSTER + pSql->index++; +#else // for single node situation, do NOT try next index +#endif pSql->thandle = NULL; // todo taos_stop_query() in async model @@ -290,24 +408,48 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { // renew meter meta in case it is changed if (pCmd->command < TSDB_SQL_FETCH && pRes->code != TSDB_CODE_QUERY_CANCELLED) { +#ifdef CLUSTER + pSql->maxRetry = TSDB_VNODES_SUPPORT * 2; +#else // for fetch, it shall not renew meter meta pSql->maxRetry = 2; - code = tscRenewMeterMeta(pSql, pCmd->name); +#endif + code = tscRenewMeterMeta(pSql, pMeterMetaInfo->name); pRes->code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return pSql; - if (pCmd->pMeterMeta) { + if (pMeterMetaInfo->pMeterMeta) { code = tscSendMsgToServer(pSql); if (code == 0) return pSql; } } } else { - if (pMsg->content[0] == TSDB_CODE_NOT_ACTIVE_SESSION || pMsg->content[0] == TSDB_CODE_NETWORK_UNAVAIL || +#ifdef CLUSTER + if (pMsg->content[0] == TSDB_CODE_REDIRECT) { + tscTrace("%p it shall be redirected!", pSql); + taosAddConnIntoCache(tscConnCache, thandle, pSql->ip, pSql->vnode, pObj->user); + pSql->thandle = NULL; + + if (pCmd->command > TSDB_SQL_MGMT) { + tscProcessMgmtRedirect(pSql, pMsg->content + 1); + } else { + pSql->index++; + } + + code = tscSendMsgToServer(pSql); + if (code == 0) return pSql; + msg = NULL; + } else if (pMsg->content[0] == TSDB_CODE_NOT_ACTIVE_SESSION || pMsg->content[0] == TSDB_CODE_NETWORK_UNAVAIL || + pMsg->content[0] == TSDB_CODE_INVALID_SESSION_ID) { +#else + if (pMsg->content[0] == TSDB_CODE_NOT_ACTIVE_SESSION || pMsg->content[0] == TSDB_CODE_NETWORK_UNAVAIL || pMsg->content[0] == TSDB_CODE_INVALID_SESSION_ID) { +#endif pSql->thandle = NULL; taosAddConnIntoCache(tscConnCache, thandle, pSql->ip, pSql->vnode, pObj->user); - if (UTIL_METER_IS_METRIC(pCmd) && pMsg->content[0] == TSDB_CODE_NOT_ACTIVE_SESSION) { + if (pMeterMetaInfo != NULL && UTIL_METER_IS_METRIC(pMeterMetaInfo) && + pMsg->content[0] == TSDB_CODE_NOT_ACTIVE_SESSION) { /* * for metric query, in case of any meter missing during query, sub-query of metric query will failed, * causing metric query failed, and return TSDB_CODE_METRICMETA_EXPIRED code to app @@ -315,12 +457,12 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { tscTrace("%p invalid meters id cause metric query failed, code:%d", pSql, pMsg->content[0]); code = TSDB_CODE_METRICMETA_EXPIRED; } else if ((pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_SELECT) && - pMsg->content[0] == TSDB_CODE_INVALID_SESSION_ID) { + pMsg->content[0] == TSDB_CODE_INVALID_SESSION_ID) { /* * session id is invalid(e.g., less than 0 or larger than maximum session per * vnode) in submit/query msg, no retry */ - code = TSDB_CODE_INVALID_QUERY_MSG; + code = TSDB_CODE_INVALID_QUERY_MSG; } else if (pCmd->command == TSDB_SQL_CONNECT) { code = TSDB_CODE_NETWORK_UNAVAIL; } else if (pCmd->command == TSDB_SQL_HB) { @@ -329,16 +471,18 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { tscTrace("%p it shall renew meter meta, code:%d", pSql, pMsg->content[0]); pSql->maxRetry = TSDB_VNODES_SUPPORT * 2; - code = tscRenewMeterMeta(pSql, pCmd->name); + code = tscRenewMeterMeta(pSql, pMeterMetaInfo->name); if (code == TSDB_CODE_ACTION_IN_PROGRESS) return pSql; - if (pCmd->pMeterMeta) { + if (pMeterMetaInfo->pMeterMeta) { code = tscSendMsgToServer(pSql); if (code == 0) return pSql; } } msg = NULL; + } else { // for other error set and return to invoker + code = pMsg->content[0]; } } @@ -346,12 +490,11 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { if (msg) { if (pCmd->command < TSDB_SQL_MGMT) { - if (UTIL_METER_IS_NOMRAL_METER(pCmd)) { - if (pCmd->pMeterMeta) // it may be deleted - pCmd->pMeterMeta->index = pSql->index; + if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { + if (pMeterMetaInfo->pMeterMeta) // it may be deleted + pMeterMetaInfo->pMeterMeta->index = pSql->index; } else { - int32_t idx = (pSql->cmd.vnodeIdx == 0) ? 0 : pSql->cmd.vnodeIdx - 1; - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pCmd->pMetricMeta, idx); + SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMeterMetaInfo->pMetricMeta, pSql->cmd.vnodeIdx); pVnodeSidList->index = pSql->index; } } else { @@ -376,11 +519,19 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { pRes->code = pMsg->content[0]; pRes->rspType = pMsg->msgType; pRes->rspLen = pMsg->msgLen - sizeof(SIntMsg); - pRes->pRsp = (char *)realloc(pRes->pRsp, pRes->rspLen); - if (pRes->rspLen) memcpy(pRes->pRsp, pMsg->content + 1, pRes->rspLen - 1); + char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen); + if (tmp == NULL) { + pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; + } else { + pRes->pRsp = tmp; + if (pRes->rspLen) { + memcpy(pRes->pRsp, pMsg->content + 1, pRes->rspLen - 1); + } + } + + // ignore the error information returned from mnode when set ignore flag in sql if (pRes->code == TSDB_CODE_DB_ALREADY_EXIST && pCmd->existsCheck && pRes->rspType == TSDB_MSG_TYPE_CREATE_DB_RSP) { - /* ignore the error information returned from mnode when set ignore flag in sql */ pRes->code = TSDB_CODE_SUCCESS; } @@ -420,9 +571,9 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { tscTrace("%p Async SQL result:%d taosres:%p", pSql, code, taosres); /* - * Whether to free sqlObj or not should be decided before call the user defined function, since - * this SqlObj may be freed in UDF, and reused by other threads before tscShouldFreeAsyncSqlObj - * called, in which case tscShouldFreeAsyncSqlObj checks an object which is actually allocated by other threads. + * Whether to free sqlObj or not should be decided before call the user defined function, since this SqlObj + * may be freed in UDF, and reused by other threads before tscShouldFreeAsyncSqlObj called, in which case + * tscShouldFreeAsyncSqlObj checks an object which is actually allocated by other threads. * * If this block of memory is re-allocated for an insert thread, in which tscKeepConn[command] equals to 0, * the tscShouldFreeAsyncSqlObj will success and tscFreeSqlObj free it immediately. @@ -450,27 +601,140 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) { return ahandle; } -static SSqlObj* tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj* prevSqlObj); +static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); static int tscLaunchMetricSubQueries(SSqlObj *pSql); -int tscProcessSql(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; +// todo merge with callback +int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, int16_t vnodeIdx, SJoinSubquerySupporter *pSupporter) { SSqlCmd *pCmd = &pSql->cmd; - tscTrace("%p SQL cmd:%d will be processed, name:%s", pSql, pSql->cmd.command, pSql->cmd.name); + pSql->res.qhandle = 0x1; + pSql->res.numOfRows = 0; + + if (pSql->pSubs == NULL) { + pSql->pSubs = malloc(POINTER_BYTES * pSupporter->pState->numOfTotal); + if (pSql->pSubs == NULL) { + return TSDB_CODE_CLI_OUT_OF_MEMORY; + } + } + + SSqlObj *pNew = createSubqueryObj(pSql, vnodeIdx, tableIndex, tscJoinQueryCallback, pSupporter, NULL); + if (pNew == NULL) { + return TSDB_CODE_CLI_OUT_OF_MEMORY; + } + + pSql->pSubs[pSql->numOfSubs++] = pNew; + if (QUERY_IS_JOIN_QUERY(pCmd->type)) { + addGroupInfoForSubquery(pSql, pNew, tableIndex); + + // refactor as one method + tscColumnBaseInfoUpdateTableIndex(&pNew->cmd.colList, 0); + tscColumnBaseInfoCopy(&pSupporter->colList, &pNew->cmd.colList, 0); + + tscSqlExprCopy(&pSupporter->exprsInfo, &pNew->cmd.exprsInfo, pSupporter->uid); + + tscFieldInfoCopyAll(&pNew->cmd.fieldsInfo, &pSupporter->fieldsInfo); + tscTagCondCopy(&pSupporter->tagCond, &pNew->cmd.tagCond); + pSupporter->groupbyExpr = pNew->cmd.groupbyExpr; + + pNew->cmd.numOfCols = 0; + pNew->cmd.nAggTimeInterval = 0; + memset(&pNew->cmd.limit, 0, sizeof(SLimitVal)); + memset(&pNew->cmd.groupbyExpr, 0, sizeof(SSqlGroupbyExpr)); + + // set the ts,tags that involved in join, as the output column of intermediate result + tscFreeSqlCmdData(&pNew->cmd); + + SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; + SColumnIndex index = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + + tscAddSpecialColumnForSelect(&pNew->cmd, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL); + + // set the tags value for ts_comp function + SSqlExpr *pExpr = tscSqlExprGet(&pNew->cmd, 0); + + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pNew->cmd, 0); + int16_t tagColIndex = tscGetJoinTagColIndexByUid(&pNew->cmd, pMeterMetaInfo->pMeterMeta->uid); + + pExpr->param->i64Key = tagColIndex; + pExpr->numOfParams = 1; + + addRequiredTagColumn(pCmd, tagColIndex, 0); + + // add the filter tag column + for (int32_t i = 0; i < pSupporter->colList.numOfCols; ++i) { + SColumnBase *pColBase = &pSupporter->colList.pColList[i]; + if (pColBase->numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. + tscColumnBaseCopy(&pNew->cmd.colList.pColList[pNew->cmd.colList.numOfCols], pColBase); + pNew->cmd.colList.numOfCols++; + } + } + } else { + pNew->cmd.type |= TSDB_QUERY_TYPE_SUBQUERY; + } + + return tscProcessSql(pNew); +} + +int doProcessSql(SSqlObj *pSql) { + SSqlCmd *pCmd = &pSql->cmd; + SSqlRes *pRes = &pSql->res; + + int32_t code = TSDB_CODE_SUCCESS; + + void *asyncFp = pSql->fp; + if (tscBuildMsg[pCmd->command](pSql) < 0) { // build msg failed + code = TSDB_CODE_APP_ERROR; + } else { + code = tscSendMsgToServer(pSql); + } + if (asyncFp) { + if (code != 0) { + pRes->code = code; + tscQueueAsyncRes(pSql); + } + return 0; + } + + if (code != 0) { + pRes->code = code; + return code; + } + + tsem_wait(&pSql->rspSem); + + if (pRes->code == 0 && tscProcessMsgRsp[pCmd->command]) (*tscProcessMsgRsp[pCmd->command])(pSql); + + tsem_post(&pSql->emptyRspSem); + + return pRes->code; +} + +int tscProcessSql(SSqlObj *pSql) { + char * name = NULL; + SSqlRes * pRes = &pSql->res; + SSqlCmd * pCmd = &pSql->cmd; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + + if (pMeterMetaInfo != NULL) { + name = pMeterMetaInfo->name; + } + + tscTrace("%p SQL cmd:%d will be processed, name:%s, type:%d", pSql, pSql->cmd.command, name, pSql->cmd.type); pSql->retry = 0; if (pSql->cmd.command < TSDB_SQL_MGMT) { +#ifdef CLUSTER + pSql->maxRetry = TSDB_VNODES_SUPPORT; +#else pSql->maxRetry = 2; - - if (UTIL_METER_IS_NOMRAL_METER(pCmd)) { - pSql->index = pCmd->pMeterMeta->index; - } else { - if (pSql->cmd.vnodeIdx == 0) { // it must be the parent SSqlObj for metric query - // do nothing - } else { - int32_t idx = pSql->cmd.vnodeIdx - 1; - SVnodeSidList *pSidList = tscGetVnodeSidList(pCmd->pMetricMeta, idx); +#endif + if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { + pSql->index = pMeterMetaInfo->pMeterMeta->index; + } else { // it must be the parent SSqlObj for metric query + if ((pSql->cmd.type & TSDB_QUERY_TYPE_SUBQUERY) != 0) { + int32_t idx = pSql->cmd.vnodeIdx; + SVnodeSidList *pSidList = tscGetVnodeSidList(pMeterMetaInfo->pMetricMeta, idx); pSql->index = pSidList->index; } } @@ -480,9 +744,53 @@ int tscProcessSql(SSqlObj *pSql) { return (*tscProcessMsgRsp[pCmd->command])(pSql); } - int code = 0; + // todo handle async situation + if (QUERY_IS_JOIN_QUERY(pSql->cmd.type)) { + if ((pSql->cmd.type & TSDB_QUERY_TYPE_SUBQUERY) == 0) { + SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); + pState->numOfTotal = pSql->cmd.numOfTables; + + for (int32_t i = 0; i < pSql->cmd.numOfTables; ++i) { + SJoinSubquerySupporter *pSupporter = tscCreateJoinSupporter(pSql, pState, i); + + if (pSupporter == NULL) { // failed to create support struct, abort current query + tscError("%p tableIndex:%d, failed to allocate join support object, abort further query", pSql, i); + pState->numOfCompleted = pSql->cmd.numOfTables - i - 1; + pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY; + + return pSql->res.code; + } + + int32_t code = tscLaunchJoinSubquery(pSql, i, 0, pSupporter); + if (code != TSDB_CODE_SUCCESS) { // failed to create subquery object, quit query + tscDestroyJoinSupporter(pSupporter); + pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY; + + break; + } + } + + sem_post(&pSql->emptyRspSem); + sem_wait(&pSql->rspSem); + + sem_post(&pSql->emptyRspSem); + + if (pSql->numOfSubs <= 0) { + pSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + } else { + pSql->cmd.command = TSDB_SQL_METRIC_JOIN_RETRIEVE; + } + + return TSDB_CODE_SUCCESS; + } else { + // for first stage sub query, iterate all vnodes to get all timestamp + if ((pSql->cmd.type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) != TSDB_QUERY_TYPE_JOIN_SEC_STAGE) { + return doProcessSql(pSql); + } + } + } - if (tscIsTwoStageMergeMetricQuery(pSql)) { // query on metric + if (tscIsTwoStageMergeMetricQuery(pCmd)) { /* * (ref. line: 964) * Before this function returns from tscLaunchMetricSubQueries and continues, pSql may have been released at user @@ -498,51 +806,48 @@ int tscProcessSql(SSqlObj *pSql) { } if (fp == NULL) { - tsem_post(&pSql->emptyRspSem); - tsem_wait(&pSql->rspSem); - - assert(pSql->cmd.vnodeIdx == 0); - tsem_post(&pSql->emptyRspSem); + sem_post(&pSql->emptyRspSem); + sem_wait(&pSql->rspSem); + sem_post(&pSql->emptyRspSem); // set the command flag must be after the semaphore been correctly set. pSql->cmd.command = TSDB_SQL_RETRIEVE_METRIC; } return pSql->res.code; - } else { - void *asyncFp = pSql->fp; - if (tscBuildMsg[pCmd->command](pSql) < 0) { // build msg failed - code = TSDB_CODE_APP_ERROR; - } else { - code = tscSendMsgToServer(pSql); - } - if (asyncFp) { - if (code != 0) { - pRes->code = code; - tscQueueAsyncRes(pSql); - } - return 0; - } } - if (code != 0) { - pRes->code = code; - return code; - } + return doProcessSql(pSql); +} - tsem_wait(&pSql->rspSem); +static void doCleanupSubqueries(SSqlObj *pSql, int32_t vnodeIndex, int32_t numOfVnodes, SRetrieveSupport *pTrs, + tOrderDescriptor *pDesc, tColModel *pModel, tExtMemBuffer **pMemoryBuf, + SSubqueryState *pState) { + pSql->cmd.command = TSDB_SQL_RETRIEVE_METRIC; + pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY; - if (pRes->code == 0 && tscProcessMsgRsp[pCmd->command]) (*tscProcessMsgRsp[pCmd->command])(pSql); + /* + * if i > 0, at least one sub query is issued, the allocated resource is + * freed by it when subquery completed. + */ + if (vnodeIndex == 0) { + tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, numOfVnodes); + tfree(pState); - tsem_post(&pSql->emptyRspSem); + if (pTrs != NULL) { + tfree(pTrs->localBuffer); - return pRes->code; + pthread_mutex_unlock(&pTrs->queryMutex); + pthread_mutex_destroy(&pTrs->queryMutex); + tfree(pTrs); + } + } } int tscLaunchMetricSubQueries(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; - /* pRes->code check only serves in launching metric sub-queries */ + // pRes->code check only serves in launching metric sub-queries if (pRes->code == TSDB_CODE_QUERY_CANCELLED) { pSql->cmd.command = TSDB_SQL_RETRIEVE_METRIC; // enable the abort of kill metric function. return pSql->res.code; @@ -554,8 +859,9 @@ int tscLaunchMetricSubQueries(SSqlObj *pSql) { pRes->qhandle = 1; // hack the qhandle check - const uint32_t nBufferSize = (1 << 16); // 64KB - int32_t numOfVnodes = pSql->cmd.pMetricMeta->numOfVnodes; + const uint32_t nBufferSize = (1 << 16); // 64KB + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0); + int32_t numOfVnodes = pMeterMetaInfo->pMetricMeta->numOfVnodes; assert(numOfVnodes > 0); int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize); @@ -571,45 +877,35 @@ int tscLaunchMetricSubQueries(SSqlObj *pSql) { pSql->numOfSubs = numOfVnodes; tscTrace("%p retrieved query data from %d vnode(s)", pSql, numOfVnodes); - int32_t * retrievedDoneRec = calloc(1, sizeof(int64_t) << 1); - int32_t * subStatusCode = &retrievedDoneRec[1]; - uint64_t *numOfTotalRetrievedPoints = (uint64_t *)&retrievedDoneRec[2]; - + SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); + pState->numOfTotal = numOfVnodes; pRes->code = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < numOfVnodes; ++i) { if (pRes->code == TSDB_CODE_QUERY_CANCELLED || pRes->code == TSDB_CODE_CLI_OUT_OF_MEMORY) { /* - * during launch sub queries, if the master query is cancelled. - * the remain is ignored and set the retrieveDoneRec to the value of remaining - * not built sub-queries. So, the already issued sub queries can successfully free allocated resources. + * during launch sub queries, if the master query is cancelled. the remain is ignored and set the retrieveDoneRec + * to the value of remaining not built sub-queries. So, the already issued sub queries can successfully free + * allocated resources. */ - *retrievedDoneRec = (numOfVnodes - i); + pState->numOfCompleted = (numOfVnodes - i); + doCleanupSubqueries(pSql, i, numOfVnodes, NULL, pDesc, pModel, pMemoryBuf, pState); if (i == 0) { - /* - * if i > 0, at least one sub query is issued, the allocated resource is done by it when it completed. - */ - tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, nBufferSize); - free(retrievedDoneRec); - pSql->cmd.command = TSDB_SQL_RETRIEVE_METRIC; - // enable the abort of kill metric function. return pSql->res.code; } + break; } SRetrieveSupport *trs = (SRetrieveSupport *)calloc(1, sizeof(SRetrieveSupport)); trs->pExtMemBuffer = pMemoryBuf; trs->pOrderDescriptor = pDesc; - trs->numOfFinished = retrievedDoneRec; - trs->code = subStatusCode; + trs->pState = pState; trs->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage)); - trs->vnodeIdx = i + 1; - trs->numOfVnodes = numOfVnodes; + trs->vnodeIdx = i; trs->pParentSqlObj = pSql; trs->pFinalColModel = pModel; - trs->numOfTotalRetrievedPoints = numOfTotalRetrievedPoints; pthread_mutexattr_t mutexattr = {0}; pthread_mutexattr_settype(&mutexattr, PTHREAD_MUTEX_RECURSIVE_NP); @@ -617,6 +913,23 @@ int tscLaunchMetricSubQueries(SSqlObj *pSql) { pthread_mutexattr_destroy(&mutexattr); SSqlObj *pNew = tscCreateSqlObjForSubquery(pSql, trs, NULL); + + if (pNew == NULL) { + pState->numOfCompleted = (numOfVnodes - i); + doCleanupSubqueries(pSql, i, numOfVnodes, trs, pDesc, pModel, pMemoryBuf, pState); + + if (i == 0) { + return pSql->res.code; + } + + break; + } + + // todo handle multi-vnode situation + if (pSql->cmd.tsBuf) { + pNew->cmd.tsBuf = tsBufClone(pSql->cmd.tsBuf); + } + tscTrace("%p sub:%p launch subquery.orderOfSub:%d", pSql, pNew, pNew->cmd.vnodeIdx); tscProcessSql(pNew); } @@ -630,6 +943,7 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { if (pSql->res.code == TSDB_CODE_SUCCESS) { taos_free_result(pSql); } + tfree(trsupport->localBuffer); pthread_mutex_unlock(&trsupport->queryMutex); @@ -638,6 +952,8 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { tfree(trsupport); } +static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); + static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t errCode) { // set no disk space error info #ifdef WINDOWS @@ -653,12 +969,12 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES tscError("sub:%p failed to flush data to disk:reason:%s", tres, buf); #endif - *(trsupport->code) = -errCode; + trsupport->pState->code = -errCode; trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; pthread_mutex_unlock(&trsupport->queryMutex); - tscRetrieveFromVnodeCallBack(trsupport, tres, *(trsupport->code)); + tscRetrieveFromVnodeCallBack(trsupport, tres, trsupport->pState->code); } static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { @@ -668,8 +984,8 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq assert(pSql != NULL); /* retrieved in subquery failed. OR query cancelled in retrieve phase. */ - if (*trsupport->code == TSDB_CODE_SUCCESS && pPObj->res.code != TSDB_CODE_SUCCESS) { - *trsupport->code = -(int)pPObj->res.code; + if (trsupport->pState->code == TSDB_CODE_SUCCESS && pPObj->res.code != TSDB_CODE_SUCCESS) { + trsupport->pState->code = -(int)pPObj->res.code; /* * kill current sub-query connection, which may retrieve data from vnodes; @@ -678,56 +994,64 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq pSql->res.numOfRows = 0; trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; // disable retry efforts tscTrace("%p query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%d", trsupport->pParentSqlObj, pSql, - trsupport->vnodeIdx, *trsupport->code); + trsupport->vnodeIdx, trsupport->pState->code); } - if (numOfRows >= 0) { - /* current query is successful, but other sub query failed, still abort current query. */ + if (numOfRows >= 0) { // current query is successful, but other sub query failed, still abort current query. tscTrace("%p sub:%p retrieve numOfRows:%d,orderOfSub:%d", pPObj, pSql, numOfRows, idx); - tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%d", - pPObj, pSql, idx, *trsupport->code); + tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%d", pPObj, pSql, idx, + trsupport->pState->code); } else { - if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && *(trsupport->code) == TSDB_CODE_SUCCESS) { + if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && trsupport->pState->code == TSDB_CODE_SUCCESS) { /* - * current query failed, and the retry count is less than the available count, - * retry query clear previous retrieved data, then launch a new sub query + * current query failed, and the retry count is less than the available + * count, retry query clear previous retrieved data, then launch a new sub query */ - tExtMemBufferClear(trsupport->pExtMemBuffer[idx - 1]); + tExtMemBufferClear(trsupport->pExtMemBuffer[idx]); // clear local saved number of results trsupport->localBuffer->numOfElems = 0; - pthread_mutex_unlock(&trsupport->queryMutex); + tscTrace("%p sub:%p retrieve failed, code:%d, orderOfSub:%d, retry:%d", trsupport->pParentSqlObj, pSql, numOfRows, + idx, trsupport->numOfRetry); + SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSqlObj, trsupport, pSql); - tscTrace("%p sub:%p retrieve failed, code:%d, orderOfSub:%d, retry:%d, new SqlObj:%p", - trsupport->pParentSqlObj, pSql, numOfRows, idx, trsupport->numOfRetry, pNew); + if (pNew == NULL) { + tscError("%p sub:%p failed to create new subquery sqlobj due to out of memory, abort retry", + trsupport->pParentSqlObj, pSql); + + trsupport->pState->code = TSDB_CODE_CLI_OUT_OF_MEMORY; + trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; + return; + } tscProcessSql(pNew); return; - } else { - /* reach the maximum retry count, abort. */ - __sync_val_compare_and_swap_32(trsupport->code, TSDB_CODE_SUCCESS, numOfRows); - tscError("%p sub:%p retrieve failed,code:%d,orderOfSub:%d failed.no more retry,set global code:%d", - pPObj, pSql, numOfRows, idx, *trsupport->code); + } else { // reach the maximum retry count, abort + __sync_val_compare_and_swap_32(&trsupport->pState->code, TSDB_CODE_SUCCESS, numOfRows); + tscError("%p sub:%p retrieve failed,code:%d,orderOfSub:%d failed.no more retry,set global code:%d", pPObj, pSql, + numOfRows, idx, trsupport->pState->code); } } - if (__sync_add_and_fetch_32(trsupport->numOfFinished, 1) < trsupport->numOfVnodes) { + if (__sync_add_and_fetch_32(&trsupport->pState->numOfCompleted, 1) < trsupport->pState->numOfTotal) { return tscFreeSubSqlObj(trsupport, pSql); } // all subqueries are failed - tscError("%p retrieve from %d vnode(s) completed,code:%d.FAILED.", pPObj, trsupport->numOfVnodes, *trsupport->code); - pPObj->res.code = -(*trsupport->code); + tscError("%p retrieve from %d vnode(s) completed,code:%d.FAILED.", pPObj, trsupport->pState->numOfTotal, + trsupport->pState->code); + pPObj->res.code = -(trsupport->pState->code); // release allocated resource tscLocalReducerEnvDestroy(trsupport->pExtMemBuffer, trsupport->pOrderDescriptor, trsupport->pFinalColModel, - trsupport->numOfVnodes); + trsupport->pState->numOfTotal); - tfree(trsupport->numOfFinished); + tfree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); + // sync query, wait for the master SSqlObj to proceed if (pPObj->fp == NULL) { // sync query, wait for the master SSqlObj to proceed tsem_wait(&pPObj->emptyRspSem); @@ -737,9 +1061,13 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; } else { - // in async query model, no need to sync operation - if (pPObj->res.code != 0) { - tscQueueAsyncRes(pPObj); + // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes + if ((pPObj->cmd.type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) == TSDB_QUERY_TYPE_JOIN_SEC_STAGE) { + (*pPObj->fp)(pPObj->param, pPObj, pPObj->res.code); + } else { // regular super table query + if (pPObj->res.code != TSDB_CODE_SUCCESS) { + tscQueueAsyncRes(pPObj); + } } } } @@ -760,22 +1088,23 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { // query process and cancel query process may execute at the same time pthread_mutex_lock(&trsupport->queryMutex); - if (numOfRows < 0 || *(trsupport->code) < 0 || pPObj->res.code != TSDB_CODE_SUCCESS) { + if (numOfRows < 0 || trsupport->pState->code < 0 || pPObj->res.code != TSDB_CODE_SUCCESS) { return tscHandleSubRetrievalError(trsupport, pSql, numOfRows); } - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; + SSqlRes * pRes = &pSql->res; + SSqlCmd * pCmd = &pSql->cmd; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); - SVnodeSidList *vnodeInfo = tscGetVnodeSidList(pCmd->pMetricMeta, idx - 1); + SVnodeSidList *vnodeInfo = tscGetVnodeSidList(pMeterMetaInfo->pMetricMeta, idx); SVPeerDesc * pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; if (numOfRows > 0) { assert(pRes->numOfRows == numOfRows); - __sync_add_and_fetch_64(trsupport->numOfTotalRetrievedPoints, numOfRows); + __sync_add_and_fetch_64(&trsupport->pState->numOfRetrievedRows, numOfRows); - tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", - pPObj, pSql, pRes->numOfRows, *trsupport->numOfTotalRetrievedPoints, pSvd->ip, pSvd->vnode, idx); + tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql, + pRes->numOfRows, trsupport->pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx); #ifdef _DEBUG_VIEW printf("received data from vnode: %d rows\n", pRes->numOfRows); @@ -784,13 +1113,12 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { tColModelDisplayEx(pDesc->pSchema, pRes->data, pRes->numOfRows, pRes->numOfRows, colInfo); #endif if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) { - tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", - pPObj, pSql, tsAvailTmpDirGB, tsMinimalTmpDirGB); + tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql, + tsAvailTmpDirGB, tsMinimalTmpDirGB); tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); return; } - - int32_t ret = saveToBuffer(trsupport->pExtMemBuffer[idx - 1], pDesc, trsupport->localBuffer, pRes->data, + int32_t ret = saveToBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pRes->data, pRes->numOfRows, pCmd->groupbyExpr.orderType); if (ret < 0) { // set no disk space error info, and abort retry @@ -800,11 +1128,12 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param); } - } else { - // all data has been retrieved to client data in from current vnode is stored in cache and disk - uint32_t numOfRowsFromVnode = trsupport->pExtMemBuffer[idx - 1]->numOfAllElems + trsupport->localBuffer->numOfElems; - tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", - pPObj, pSql, pSvd->ip, pSvd->vnode, numOfRowsFromVnode, idx); + } else { // all data has been retrieved to client + /* data in from current vnode is stored in cache and disk */ + uint32_t numOfRowsFromVnode = + trsupport->pExtMemBuffer[pCmd->vnodeIdx]->numOfAllElems + trsupport->localBuffer->numOfElems; + tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip, + pSvd->vnode, numOfRowsFromVnode, idx); tColModelCompact(pDesc->pSchema, trsupport->localBuffer, pDesc->pSchema->maxCapacity); @@ -816,33 +1145,33 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { trsupport->localBuffer->numOfElems, colInfo); #endif if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) { - tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", - pPObj, pSql, tsAvailTmpDirGB, tsMinimalTmpDirGB); + tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql, + tsAvailTmpDirGB, tsMinimalTmpDirGB); tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); return; } // each result for a vnode is ordered as an independant list, // then used as an input of loser tree for disk-based merge routine - int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx - 1], pDesc, trsupport->localBuffer, - pCmd->groupbyExpr.orderType); + int32_t ret = + tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pCmd->groupbyExpr.orderType); if (ret != 0) { /* set no disk space error info, and abort retry */ return tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); } - if (__sync_add_and_fetch_32(trsupport->numOfFinished, 1) < trsupport->numOfVnodes) { + if (__sync_add_and_fetch_32(&trsupport->pState->numOfCompleted, 1) < trsupport->pState->numOfTotal) { return tscFreeSubSqlObj(trsupport, pSql); } // all sub-queries are returned, start to local merge process - pDesc->pSchema->maxCapacity = trsupport->pExtMemBuffer[idx - 1]->numOfElemsPerPage; + pDesc->pSchema->maxCapacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; - tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", - pPObj, trsupport->numOfVnodes, *trsupport->numOfTotalRetrievedPoints); + tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj, + trsupport->pState->numOfTotal, trsupport->pState->numOfCompleted); tscClearInterpInfo(&pPObj->cmd); - tscCreateLocalReducer(trsupport->pExtMemBuffer, trsupport->numOfVnodes, pDesc, trsupport->pFinalColModel, + tscCreateLocalReducer(trsupport->pExtMemBuffer, trsupport->pState->numOfTotal, pDesc, trsupport->pFinalColModel, &pPObj->cmd, &pPObj->res); tscTrace("%p build loser tree completed", pPObj); @@ -851,7 +1180,7 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { pPObj->res.row = 0; // only free once - free(trsupport->numOfFinished); + free(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); if (pPObj->fp == NULL) { @@ -872,7 +1201,7 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { } void tscKillMetricQuery(SSqlObj *pSql) { - if (!tscIsTwoStageMergeMetricQuery(pSql)) { + if (!tscIsTwoStageMergeMetricQuery(&pSql->cmd)) { return; } @@ -882,6 +1211,7 @@ void tscKillMetricQuery(SSqlObj *pSql) { if (pSub == NULL || pSub->thandle == NULL) { continue; } + /* * here, we cannot set the command = TSDB_SQL_KILL_QUERY. Otherwise, it may cause * sub-queries not correctly released and master sql object of metric query reaches an abnormal state. @@ -911,107 +1241,84 @@ void tscKillMetricQuery(SSqlObj *pSql) { tscTrace("%p metric query is cancelled", pSql); } -SSqlObj* tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj* prevSqlObj) { - SSqlCmd *pCmd = &pSql->cmd; - - SSqlObj *pNew = (SSqlObj *)calloc(1, sizeof(SSqlObj)); - if (pNew == NULL) { - return NULL; - } - - pSql->pSubs[trsupport->vnodeIdx - 1] = pNew; - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->sqlstr = strdup(pSql->sqlstr); - - memcpy(&pNew->cmd, pCmd, sizeof(SSqlCmd)); - pNew->cmd.command = TSDB_SQL_SELECT; - pNew->cmd.payload = NULL; - pNew->cmd.allocSize = 0; +static void tscRetrieveDataRes(void *param, TAOS_RES *tres, int retCode); - tscTagCondAssign(&pNew->cmd.tagCond, &pCmd->tagCond); - - tscAllocPayloadWithSize(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE); - tscColumnInfoClone(&pCmd->colList, &pNew->cmd.colList); - tscFieldInfoClone(&pCmd->fieldsInfo, &pNew->cmd.fieldsInfo); - tscSqlExprClone(&pCmd->exprsInfo, &pNew->cmd.exprsInfo); - - pNew->fp = tscRetrieveDataRes; - - pNew->param = trsupport; - pNew->cmd.vnodeIdx = trsupport->vnodeIdx; - - if (prevSqlObj == NULL) { - char key[TSDB_MAX_TAGS_LEN + 1] = {0}; - tscGetMetricMetaCacheKey(&pNew->cmd, key); - pNew->cmd.pMetricMeta = taosGetDataFromCache(tscCacheHandle, key); - pNew->cmd.pMeterMeta = taosGetDataFromCache(tscCacheHandle, pCmd->name); - } else { - pNew->cmd.pMeterMeta = prevSqlObj->cmd.pMeterMeta; - pNew->cmd.pMetricMeta = prevSqlObj->cmd.pMetricMeta; - - prevSqlObj->cmd.pMetricMeta = NULL; - prevSqlObj->cmd.pMeterMeta = NULL; +static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) { + SSqlObj *pNew = createSubqueryObj(pSql, trsupport->vnodeIdx, 0, tscRetrieveDataRes, trsupport, prevSqlObj); + if (pNew != NULL) { // the sub query of two-stage super table query + pNew->cmd.type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY; + pSql->pSubs[trsupport->vnodeIdx] = pNew; } - assert(pNew->cmd.pMeterMeta != NULL && pNew->cmd.pMetricMeta != NULL); return pNew; } -void tscRetrieveDataRes(void *param, TAOS_RES *tres, int retCode) { +void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { SRetrieveSupport *trsupport = (SRetrieveSupport *)param; - SSqlObj *pSql = (SSqlObj *)tres; - int32_t idx = pSql->cmd.vnodeIdx; + SSqlObj * pSql = (SSqlObj *)tres; + int32_t idx = pSql->cmd.vnodeIdx; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0); SVnodeSidList *vnodeInfo = NULL; - if (pSql->cmd.pMetricMeta != NULL) { - vnodeInfo = tscGetVnodeSidList(pSql->cmd.pMetricMeta, idx - 1); + SVPeerDesc * pSvd = NULL; + if (pMeterMetaInfo->pMetricMeta != NULL) { + vnodeInfo = tscGetVnodeSidList(pMeterMetaInfo->pMetricMeta, idx); + pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; } - if (trsupport->pParentSqlObj->res.code != TSDB_CODE_SUCCESS || *trsupport->code != TSDB_CODE_SUCCESS) { - // metric query is killed, Note: retCode must be less than 0 + if (trsupport->pParentSqlObj->res.code != TSDB_CODE_SUCCESS || trsupport->pState->code != TSDB_CODE_SUCCESS) { + // metric query is killed, Note: code must be less than 0 trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; if (trsupport->pParentSqlObj->res.code != TSDB_CODE_SUCCESS) { - retCode = -(int)(trsupport->pParentSqlObj->res.code); + code = -(int)(trsupport->pParentSqlObj->res.code); } else { - retCode = (*trsupport->code); + code = trsupport->pState->code; } tscTrace("%p query cancelled or failed, sub:%p, orderOfSub:%d abort, code:%d", trsupport->pParentSqlObj, pSql, - trsupport->vnodeIdx, retCode); + trsupport->vnodeIdx, code); } /* - * if a query on vnode is failed, all retrieve operations from vnode that occurs later + * if a query on a vnode is failed, all retrieve operations from vnode that occurs later * than this one are actually not necessary, we simply call the tscRetrieveFromVnodeCallBack * function to abort current and remain retrieve process. - * Note: threadsafe is required. + * + * NOTE: threadsafe is required. */ - if (retCode != TSDB_CODE_SUCCESS) { + if (code != TSDB_CODE_SUCCESS) { if (trsupport->numOfRetry++ >= MAX_NUM_OF_SUBQUERY_RETRY) { - tscTrace("%p sub:%p reach the max retry count,set global code:%d", trsupport->pParentSqlObj, pSql, retCode); - __sync_val_compare_and_swap_32(trsupport->code, 0, retCode); + tscTrace("%p sub:%p reach the max retry count,set global code:%d", trsupport->pParentSqlObj, pSql, code); + __sync_val_compare_and_swap_32(&trsupport->pState->code, 0, code); } else { // does not reach the maximum retry count, go on + tscTrace("%p sub:%p failed code:%d, retry:%d", trsupport->pParentSqlObj, pSql, code, trsupport->numOfRetry); + SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSqlObj, trsupport, pSql); - tscTrace("%p sub:%p failed code:%d, retry:%d, new SqlObj:%p", trsupport->pParentSqlObj, pSql, retCode, - trsupport->numOfRetry, pNew); + if (pNew == NULL) { + tscError("%p sub:%p failed to create new subquery due to out of memory, abort retry, vid:%d, orderOfSub:%d", + trsupport->pParentSqlObj, pSql, pSvd->vnode, trsupport->vnodeIdx); - tscProcessSql(pNew); - return; + trsupport->pState->code = -TSDB_CODE_CLI_OUT_OF_MEMORY; + trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; + } else { + assert(pNew->cmd.pMeterInfo[0]->pMeterMeta != NULL && pNew->cmd.pMeterInfo[0]->pMetricMeta != NULL); + tscProcessSql(pNew); + return; + } } } - if (*(trsupport->code) != TSDB_CODE_SUCCESS) { // failed, abort + if (trsupport->pState->code != TSDB_CODE_SUCCESS) { // failed, abort if (vnodeInfo != NULL) { tscTrace("%p sub:%p query failed,ip:%u,vid:%d,orderOfSub:%d,global code:%d", trsupport->pParentSqlObj, pSql, vnodeInfo->vpeerDesc[vnodeInfo->index].ip, vnodeInfo->vpeerDesc[vnodeInfo->index].vnode, - trsupport->vnodeIdx, *(trsupport->code)); + trsupport->vnodeIdx, trsupport->pState->code); } else { tscTrace("%p sub:%p query failed,orderOfSub:%d,global code:%d", trsupport->pParentSqlObj, pSql, - trsupport->vnodeIdx, *(trsupport->code)); + trsupport->vnodeIdx, trsupport->pState->code); } - tscRetrieveFromVnodeCallBack(param, tres, *(trsupport->code)); + tscRetrieveFromVnodeCallBack(param, tres, trsupport->pState->code); } else { // success, proceed to retrieve data from dnode tscTrace("%p sub:%p query complete,ip:%u,vid:%d,orderOfSub:%d,retrieve data", trsupport->pParentSqlObj, pSql, vnodeInfo->vpeerDesc[vnodeInfo->index].ip, vnodeInfo->vpeerDesc[vnodeInfo->index].vnode, @@ -1029,9 +1336,10 @@ int tscBuildRetrieveMsg(SSqlObj *pSql) { pMsg = pStart; *((uint64_t *)pMsg) = pSql->res.qhandle; - pMsg += 8; - *pMsg = pSql->cmd.type; - pMsg += 1; + pMsg += sizeof(pSql->res.qhandle); + + *pMsg = htons(pSql->cmd.type); + pMsg += sizeof(pSql->cmd.type); msgLen = pMsg - pStart; pSql->cmd.payloadLen = msgLen; @@ -1040,10 +1348,12 @@ int tscBuildRetrieveMsg(SSqlObj *pSql) { return msgLen; } -void tscUpdateVnodeInSubmitMsg(SSqlObj *pSql, char* buf) { +void tscUpdateVnodeInSubmitMsg(SSqlObj *pSql, char *buf) { SShellSubmitMsg *pShellMsg; char * pMsg; - SMeterMeta * pMeterMeta = pSql->cmd.pMeterMeta; + SMeterMetaInfo * pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0); + + SMeterMeta *pMeterMeta = pMeterMetaInfo->pMeterMeta; pMsg = buf + tsRpcHeadSize; @@ -1056,7 +1366,9 @@ int tscBuildSubmitMsg(SSqlObj *pSql) { SShellSubmitMsg *pShellMsg; char * pMsg, *pStart; int msgLen = 0; - SMeterMeta * pMeterMeta = pSql->cmd.pMeterMeta; + + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0); + SMeterMeta * pMeterMeta = pMeterMetaInfo->pMeterMeta; pStart = pSql->cmd.payload + tsRpcHeadSize; pMsg = pStart; @@ -1064,29 +1376,28 @@ int tscBuildSubmitMsg(SSqlObj *pSql) { pShellMsg = (SShellSubmitMsg *)pMsg; pShellMsg->import = pSql->cmd.order.order; pShellMsg->vnode = htons(pMeterMeta->vpeerDesc[pMeterMeta->index].vnode); - pShellMsg->numOfSid = htonl(pSql->cmd.count); /* number of meters to be inserted */ + pShellMsg->numOfSid = htonl(pSql->cmd.count); // number of meters to be inserted - /* - * pSql->cmd.payloadLen is set during parse sql routine, so we do not use it here - */ + // pSql->cmd.payloadLen is set during parse sql routine, so we do not use it here pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; tscTrace("%p update submit msg vnode:%d", pSql, htons(pShellMsg->vnode)); return msgLen; } -void tscUpdateVnodeInQueryMsg(SSqlObj *pSql, char* buf) { - SSqlCmd *pCmd = &pSql->cmd; - char * pStart = buf + tsRpcHeadSize; +void tscUpdateVnodeInQueryMsg(SSqlObj *pSql, char *buf) { + SSqlCmd * pCmd = &pSql->cmd; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + char * pStart = buf + tsRpcHeadSize; SQueryMeterMsg *pQueryMsg = (SQueryMeterMsg *)pStart; - if (UTIL_METER_IS_NOMRAL_METER(pCmd)) { // pSchema == NULL, query on meter - SMeterMeta *pMeterMeta = pCmd->pMeterMeta; + if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { // pSchema == NULL, query on meter + SMeterMeta *pMeterMeta = pMeterMetaInfo->pMeterMeta; pQueryMsg->vnode = htons(pMeterMeta->vpeerDesc[pSql->index].vnode); } else { // query on metric - SMetricMeta * pMetricMeta = pCmd->pMetricMeta; - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pCmd->vnodeIdx - 1); + SMetricMeta * pMetricMeta = pMeterMetaInfo->pMetricMeta; + SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pCmd->vnodeIdx); pQueryMsg->vnode = htons(pVnodeSidList->vpeerDesc[pSql->index].vnode); } } @@ -1097,58 +1408,69 @@ void tscUpdateVnodeInQueryMsg(SSqlObj *pSql, char* buf) { */ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd) { const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; - int32_t srcColListSize = pCmd->numOfCols * sizeof(SColumnFilterMsg); + int32_t srcColListSize = pCmd->numOfCols * sizeof(SColumnInfo); - int32_t exprSize = sizeof(SSqlFuncExprMsg) * pCmd->fieldsInfo.numOfOutputCols; + int32_t exprSize = sizeof(SSqlFuncExprMsg) * pCmd->fieldsInfo.numOfOutputCols; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); // meter query without tags values - if (!UTIL_METER_IS_METRIC(pCmd)) { + if (!UTIL_METER_IS_METRIC(pMeterMetaInfo)) { return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryMeterMsg) + srcColListSize + exprSize; } - SMetricMeta *pMetricMeta = pCmd->pMetricMeta; + SMetricMeta *pMetricMeta = pMeterMetaInfo->pMetricMeta; - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pCmd->vnodeIdx - 1); + SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pCmd->vnodeIdx); int32_t meterInfoSize = (pMetricMeta->tagLen + sizeof(SMeterSidExtInfo)) * pVnodeSidList->numOfSids; int32_t outputColumnSize = pCmd->fieldsInfo.numOfOutputCols * sizeof(SSqlFuncExprMsg); - return meterInfoSize + outputColumnSize + srcColListSize + exprSize + MIN_QUERY_MSG_PKT_SIZE; + int32_t size = meterInfoSize + outputColumnSize + srcColListSize + exprSize + MIN_QUERY_MSG_PKT_SIZE; + if (pCmd->tsBuf != NULL) { + size += pCmd->tsBuf->fileSize; + } + + return size; } int tscBuildQueryMsg(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; int32_t size = tscEstimateQueryMsgSize(pCmd); - tscAllocPayloadWithSize(pCmd, size); - char *pStart = pCmd->payload + tsRpcHeadSize; + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + tscError("%p failed to malloc for query msg", pSql); + return -1; + } + + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + char * pStart = pCmd->payload + tsRpcHeadSize; - SMeterMeta * pMeterMeta = pCmd->pMeterMeta; - SMetricMeta *pMetricMeta = pCmd->pMetricMeta; + SMeterMeta * pMeterMeta = pMeterMetaInfo->pMeterMeta; + SMetricMeta *pMetricMeta = pMeterMetaInfo->pMetricMeta; SQueryMeterMsg *pQueryMsg = (SQueryMeterMsg *)pStart; int32_t msgLen = 0; int32_t numOfMeters = 0; - if (UTIL_METER_IS_NOMRAL_METER(pCmd)) { // pSchema == NULL, query on meter + if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { numOfMeters = 1; tscTrace("%p query on vnode: %d, number of sid:%d, meter id: %s", pSql, - pMeterMeta->vpeerDesc[pMeterMeta->index].vnode, 1, pCmd->name); + pMeterMeta->vpeerDesc[pMeterMeta->index].vnode, 1, pMeterMetaInfo->name); pQueryMsg->vnode = htons(pMeterMeta->vpeerDesc[pMeterMeta->index].vnode); pQueryMsg->uid = pMeterMeta->uid; pQueryMsg->numOfTagsCols = 0; } else { // query on metric - SMetricMeta *pMetricMeta = pCmd->pMetricMeta; - if (pCmd->vnodeIdx <= 0) { + SMetricMeta *pMetricMeta = pMeterMetaInfo->pMetricMeta; + if (pCmd->vnodeIdx < 0) { tscError("%p error vnodeIdx:%d", pSql, pCmd->vnodeIdx); return -1; } - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pCmd->vnodeIdx - 1); + SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pCmd->vnodeIdx); uint32_t vnodeId = pVnodeSidList->vpeerDesc[pVnodeSidList->index].vnode; numOfMeters = pVnodeSidList->numOfSids; @@ -1162,7 +1484,7 @@ int tscBuildQueryMsg(SSqlObj *pSql) { } pQueryMsg->numOfSids = htonl(numOfMeters); - pQueryMsg->numOfTagsCols = htons(pCmd->numOfReqTags); + pQueryMsg->numOfTagsCols = htons(pMeterMetaInfo->numOfTags); if (pCmd->order.order == TSQL_SO_ASC) { pQueryMsg->skey = htobe64(pCmd->stime); @@ -1200,25 +1522,21 @@ int tscBuildQueryMsg(SSqlObj *pSql) { return -1; } - if (UTIL_METER_IS_NOMRAL_METER(pCmd)) { // query on meter - assert(pCmd->groupbyExpr.numOfGroupbyCols == 0); + if (pCmd->groupbyExpr.numOfGroupCols < 0) { + tscError("%p illegal value of numOfGroupCols in query msg: %d", pSql, pCmd->groupbyExpr.numOfGroupCols); + return -1; + } + + pQueryMsg->numOfGroupCols = htons(pCmd->groupbyExpr.numOfGroupCols); + + if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { // query on meter pQueryMsg->tagLength = 0; } else { // query on metric - - if (pCmd->groupbyExpr.numOfGroupbyCols > 0) { - pQueryMsg->numOfGroupbyCols = htons(pCmd->groupbyExpr.numOfGroupbyCols); - if (pCmd->groupbyExpr.numOfGroupbyCols < 0) { - tscError("%p illegal value of numOfGroupbyCols in query msg: %d", pSql, pCmd->groupbyExpr.numOfGroupbyCols); - return -1; - } - } else { // no group by clause - pQueryMsg->numOfGroupbyCols = 0; - } pQueryMsg->tagLength = htons(pMetricMeta->tagLen); } - pQueryMsg->metricQuery = htons(pCmd->metricQuery); - pQueryMsg->numOfOutputCols = htons(pCmd->fieldsInfo.numOfOutputCols); + pQueryMsg->queryType = htons(pCmd->type); + pQueryMsg->numOfOutputCols = htons(pCmd->exprsInfo.numOfExprs); if (pCmd->fieldsInfo.numOfOutputCols < 0) { tscError("%p illegal value of number of output columns in query msg: %d", pSql, pCmd->fieldsInfo.numOfOutputCols); @@ -1226,56 +1544,63 @@ int tscBuildQueryMsg(SSqlObj *pSql) { } // set column list ids - char *pMsg = (char *)(pQueryMsg->colList); - char *pBinaryBuf = pMsg + sizeof(pQueryMsg->colList[0]) * pCmd->colList.numOfCols; - + char * pMsg = (char *)(pQueryMsg->colList) + pCmd->colList.numOfCols * sizeof(SColumnInfo); SSchema *pSchema = tsGetSchema(pMeterMeta); for (int32_t i = 0; i < pCmd->colList.numOfCols; ++i) { - SColumnBase *pCol = tscColumnInfoGet(pCmd, i); - SSchema * pColSchema = &pSchema[pCol->colIndex]; + SColumnBase *pCol = tscColumnBaseInfoGet(&pCmd->colList, i); + SSchema * pColSchema = &pSchema[pCol->colIndex.columnIndex]; - if (pCol->colIndex >= pMeterMeta->numOfColumns || pColSchema->type < TSDB_DATA_TYPE_BOOL || + if (pCol->colIndex.columnIndex >= pMeterMeta->numOfColumns || pColSchema->type < TSDB_DATA_TYPE_BOOL || pColSchema->type > TSDB_DATA_TYPE_NCHAR) { - tscError("%p vid:%d sid:%d id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", - pSql, htons(pQueryMsg->vnode), pMeterMeta->sid, pCmd->name, pMeterMeta->numOfColumns, pCol->colIndex, + tscError("%p vid:%d sid:%d id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", pSql, + htons(pQueryMsg->vnode), pMeterMeta->sid, pMeterMetaInfo->name, pMeterMeta->numOfColumns, pCol->colIndex, pColSchema->name); - return 0; // 0 means build msg failed + return -1; // 0 means build msg failed } pQueryMsg->colList[i].colId = htons(pColSchema->colId); pQueryMsg->colList[i].bytes = htons(pColSchema->bytes); - pQueryMsg->colList[i].type = htons(pColSchema->type); + pQueryMsg->colList[i].numOfFilters = htons(pCol->numOfFilters); - pQueryMsg->colList[i].filterOn = htons(pCol->filterOn); - pQueryMsg->colList[i].filterOnBinary = htons(pCol->filterOnBinary); + // append the filter information after the basic column information + for (int32_t f = 0; f < pCol->numOfFilters; ++f) { + SColumnFilterInfo *pColFilter = &pCol->filterInfo[f]; - if (pCol->filterOn && pCol->filterOnBinary) { - pQueryMsg->colList[i].len = htobe64(pCol->len); - memcpy(pBinaryBuf, (void *)pCol->pz, pCol->len + 1); - pBinaryBuf += pCol->len + 1; - } else { - pQueryMsg->colList[i].lowerBndi = htobe64(pCol->lowerBndi); - pQueryMsg->colList[i].upperBndi = htobe64(pCol->upperBndi); - } + SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)pMsg; + pFilterMsg->filterOnBinary = htons(pColFilter->filterOnBinary); + + pMsg += sizeof(SColumnFilterInfo); - pQueryMsg->colList[i].lowerRelOptr = htons(pCol->lowerRelOptr); - pQueryMsg->colList[i].upperRelOptr = htons(pCol->upperRelOptr); + if (pColFilter->filterOnBinary) { + pFilterMsg->len = htobe64(pColFilter->len); + memcpy(pMsg, (void *)pColFilter->pz, pColFilter->len + 1); + pMsg += (pColFilter->len + 1); // append the additional filter binary info + } else { + pFilterMsg->lowerBndi = htobe64(pColFilter->lowerBndi); + pFilterMsg->upperBndi = htobe64(pColFilter->upperBndi); + } + + pFilterMsg->lowerRelOptr = htons(pColFilter->lowerRelOptr); + pFilterMsg->upperRelOptr = htons(pColFilter->upperRelOptr); - pMsg += sizeof(SColumnFilterMsg); + if (pColFilter->lowerRelOptr == TSDB_RELATION_INVALID && pColFilter->upperRelOptr == TSDB_RELATION_INVALID) { + tscError("invalid filter info"); + return -1; + } + } } bool hasArithmeticFunction = false; - pMsg = pBinaryBuf; - SSqlFuncExprMsg *pSqlFuncExpr = (SSqlFuncExprMsg *)pBinaryBuf; + SSqlFuncExprMsg *pSqlFuncExpr = (SSqlFuncExprMsg *)pMsg; for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pCmd, i); - if (pExpr->sqlFuncId == TSDB_FUNC_ARITHM) { + if (pExpr->functionId == TSDB_FUNC_ARITHM) { hasArithmeticFunction = true; } @@ -1287,9 +1612,9 @@ int tscBuildQueryMsg(SSqlObj *pSql) { pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId); pSqlFuncExpr->colInfo.colIdx = htons(pExpr->colInfo.colIdx); - pSqlFuncExpr->colInfo.isTag = pExpr->colInfo.isTag; + pSqlFuncExpr->colInfo.flag = htons(pExpr->colInfo.flag); - pSqlFuncExpr->functionId = htons(pExpr->sqlFuncId); + pSqlFuncExpr->functionId = htons(pExpr->functionId); pSqlFuncExpr->numOfParams = htons(pExpr->numOfParams); pMsg += sizeof(SSqlFuncExprMsg); @@ -1299,7 +1624,9 @@ int tscBuildQueryMsg(SSqlObj *pSql) { if (pExpr->param[j].nType == TSDB_DATA_TYPE_BINARY) { memcpy(pMsg, pExpr->param[j].pz, pExpr->param[j].nLen); - pMsg += pExpr->param[j].nLen + 1; // by plus one char to make the string null-terminated + + // by plus one char to make the string null-terminated + pMsg += pExpr->param[j].nLen + 1; } else { pSqlFuncExpr->arg[j].argValue.i64 = htobe64(pExpr->param[j].i64Key); } @@ -1312,7 +1639,7 @@ int tscBuildQueryMsg(SSqlObj *pSql) { if (hasArithmeticFunction) { SColumnBase *pColBase = pCmd->colList.pColList; for (int32_t i = 0; i < pCmd->colList.numOfCols; ++i) { - char * name = pSchema[pColBase[i].colIndex].name; + char * name = pSchema[pColBase[i].colIndex.columnIndex].name; int32_t lenx = strlen(name); memcpy(pMsg, name, lenx); *(pMsg + lenx) = ','; @@ -1326,15 +1653,16 @@ int tscBuildQueryMsg(SSqlObj *pSql) { // set sids list tscTrace("%p vid:%d, query on %d meters", pSql, pSql->cmd.vnodeIdx, numOfMeters); - if (UTIL_METER_IS_NOMRAL_METER(pCmd)) { + if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { #ifdef _DEBUG_VIEW - tscTrace("%p %d", pSql, pCmd->pMeterMeta->sid); + + tscTrace("%p %d", pSql, pMeterMetaInfo->pMeterMeta->sid); #endif SMeterSidExtInfo *pSMeterTagInfo = (SMeterSidExtInfo *)pMsg; pSMeterTagInfo->sid = htonl(pMeterMeta->sid); pMsg += sizeof(SMeterSidExtInfo); } else { - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pCmd->vnodeIdx - 1); + SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pCmd->vnodeIdx); for (int32_t i = 0; i < numOfMeters; ++i) { SMeterSidExtInfo *pMeterTagInfo = (SMeterSidExtInfo *)pMsg; @@ -1352,16 +1680,18 @@ int tscBuildQueryMsg(SSqlObj *pSql) { } } - /* only include the required tag column schema. If a tag is not required, it won't be sent to vnode */ - if (UTIL_METER_IS_METRIC(pCmd) && pCmd->numOfReqTags > 0) { // always transfer tag schema to vnode if exists + // only include the required tag column schema. If a tag is not required, it won't be sent to vnode + if (pMeterMetaInfo->numOfTags > 0) { + // always transfer tag schema to vnode if exists SSchema *pTagSchema = tsGetTagSchema(pMeterMeta); - for (int32_t j = 0; j < pCmd->numOfReqTags; ++j) { - if (pCmd->tagColumnIndex[j] == -1) { - SSchema tbSchema = {.bytes = TSDB_METER_NAME_LEN, .colId = -1, .type = TSDB_DATA_TYPE_BINARY}; + for (int32_t j = 0; j < pMeterMetaInfo->numOfTags; ++j) { + if (pMeterMetaInfo->tagColumnIndex[j] == TSDB_TBNAME_COLUMN_INDEX) { + SSchema tbSchema = { + .bytes = TSDB_METER_NAME_LEN, .colId = TSDB_TBNAME_COLUMN_INDEX, .type = TSDB_DATA_TYPE_BINARY}; memcpy(pMsg, &tbSchema, sizeof(SSchema)); } else { - memcpy(pMsg, &pTagSchema[pCmd->tagColumnIndex[j]], sizeof(SSchema)); + memcpy(pMsg, &pTagSchema[pMeterMetaInfo->tagColumnIndex[j]], sizeof(SSchema)); } pMsg += sizeof(SSchema); @@ -1369,23 +1699,56 @@ int tscBuildQueryMsg(SSqlObj *pSql) { } SSqlGroupbyExpr *pGroupbyExpr = &pCmd->groupbyExpr; - if (pGroupbyExpr->numOfGroupbyCols != 0) { - assert(pMeterMeta->numOfTags != 0); - - pQueryMsg->orderByIdx = htons(pGroupbyExpr->orderIdx); + if (pGroupbyExpr->numOfGroupCols != 0) { + pQueryMsg->orderByIdx = htons(pGroupbyExpr->orderIndex); pQueryMsg->orderType = htons(pGroupbyExpr->orderType); - for (int32_t j = 0; j < pGroupbyExpr->numOfGroupbyCols; ++j) { - *((int16_t *)pMsg) = pGroupbyExpr->tagIndex[j]; - pMsg += sizeof(pGroupbyExpr->tagIndex[j]); - } + for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { + SColIndexEx *pCol = &pGroupbyExpr->columnInfo[j]; + + *((int16_t *)pMsg) = pCol->colId; + pMsg += sizeof(pCol->colId); + + *((int16_t *)pMsg) += pCol->colIdx; + pMsg += sizeof(pCol->colIdx); + + *((int16_t *)pMsg) += pCol->colIdxInBuf; + pMsg += sizeof(pCol->colIdxInBuf); + + *((int16_t *)pMsg) += pCol->flag; + pMsg += sizeof(pCol->flag); + } + } + + if (pCmd->interpoType != TSDB_INTERPO_NONE) { + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + *((int64_t *)pMsg) = htobe64(pCmd->defaultVal[i]); + pMsg += sizeof(pCmd->defaultVal[0]); + } + } + + // compressed ts block + pQueryMsg->tsOffset = htonl(pMsg - pStart); + int32_t tsLen = 0; + int32_t numOfBlocks = 0; + + if (pCmd->tsBuf != NULL) { + STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pCmd->tsBuf, pCmd->vnodeIdx); + assert(QUERY_IS_JOIN_QUERY(pCmd->type) && pBlockInfo != NULL); // this query should not be sent + + // todo refactor + fseek(pCmd->tsBuf->f, pBlockInfo->offset, SEEK_SET); + fread(pMsg, pBlockInfo->compLen, 1, pCmd->tsBuf->f); + + pMsg += pBlockInfo->compLen; + tsLen = pBlockInfo->compLen; + numOfBlocks = pBlockInfo->numOfBlocks; } - if (pCmd->interpoType != TSDB_INTERPO_NONE) { - for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { - *((int64_t *)pMsg) = htobe64(pCmd->defaultVal[i]); - pMsg += sizeof(pCmd->defaultVal[0]); - } + pQueryMsg->tsLen = htonl(tsLen); + pQueryMsg->tsNumOfBlocks = htonl(numOfBlocks); + if (pCmd->tsBuf != NULL) { + pQueryMsg->tsOrder = htonl(pCmd->tsBuf->tsOrder); } msgLen = pMsg - pStart; @@ -1403,8 +1766,10 @@ int tscBuildCreateDbMsg(SSqlObj *pSql) { char * pMsg, *pStart; int msgLen = 0; - SSqlCmd *pCmd = &pSql->cmd; - STscObj *pObj = pSql->pTscObj; + SSqlCmd * pCmd = &pSql->cmd; + STscObj * pObj = pSql->pTscObj; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + pStart = pCmd->payload + tsRpcHeadSize; pMsg = pStart; @@ -1413,7 +1778,7 @@ int tscBuildCreateDbMsg(SSqlObj *pSql) { pMsg += sizeof(SMgmtHead); pCreateDbMsg = (SCreateDbMsg *)pMsg; - strcpy(pCreateDbMsg->db, pCmd->name); + strcpy(pCreateDbMsg->db, pMeterMetaInfo->name); pMsg += sizeof(SCreateDbMsg); msgLen = pMsg - pStart; @@ -1428,8 +1793,10 @@ int tscBuildCreateDnodeMsg(SSqlObj *pSql) { char * pMsg, *pStart; int msgLen = 0; - SSqlCmd *pCmd = &pSql->cmd; - STscObj *pObj = pSql->pTscObj; + SSqlCmd * pCmd = &pSql->cmd; + STscObj * pObj = pSql->pTscObj; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + pMsg = pCmd->payload + tsRpcHeadSize; pStart = pMsg; @@ -1438,7 +1805,7 @@ int tscBuildCreateDnodeMsg(SSqlObj *pSql) { pMsg += sizeof(SMgmtHead); pCreate = (SCreateDnodeMsg *)pMsg; - strcpy(pCreate->ip, pCmd->name); + strcpy(pCreate->ip, pMeterMetaInfo->name); pMsg += sizeof(SCreateDnodeMsg); @@ -1454,8 +1821,10 @@ int tscBuildDropDnodeMsg(SSqlObj *pSql) { char * pMsg, *pStart; int msgLen = 0; - SSqlCmd *pCmd = &pSql->cmd; - STscObj *pObj = pSql->pTscObj; + SSqlCmd * pCmd = &pSql->cmd; + STscObj * pObj = pSql->pTscObj; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + pMsg = pCmd->payload + tsRpcHeadSize; pStart = pMsg; @@ -1464,7 +1833,7 @@ int tscBuildDropDnodeMsg(SSqlObj *pSql) { pMsg += sizeof(SMgmtHead); pDrop = (SDropDnodeMsg *)pMsg; - strcpy(pDrop->ip, pCmd->name); + strcpy(pDrop->ip, pMeterMetaInfo->name); pMsg += sizeof(SDropDnodeMsg); @@ -1480,8 +1849,10 @@ int tscBuildCreateUserMsg(SSqlObj *pSql) { char * pMsg, *pStart; int msgLen = 0; - SSqlCmd *pCmd = &pSql->cmd; - STscObj *pObj = pSql->pTscObj; + SSqlCmd * pCmd = &pSql->cmd; + STscObj * pObj = pSql->pTscObj; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + pMsg = pCmd->payload + tsRpcHeadSize; pStart = pMsg; @@ -1490,7 +1861,7 @@ int tscBuildCreateUserMsg(SSqlObj *pSql) { pMsg += sizeof(SMgmtHead); pCreateMsg = (SCreateUserMsg *)pMsg; - strcpy(pCreateMsg->user, pCmd->name); + strcpy(pCreateMsg->user, pMeterMetaInfo->name); strcpy(pCreateMsg->pass, pCmd->payload); pMsg += sizeof(SCreateUserMsg); @@ -1507,8 +1878,10 @@ static int tscBuildAcctMsgImpl(SSqlObj *pSql) { char * pMsg, *pStart; int msgLen = 0; - SSqlCmd *pCmd = &pSql->cmd; - STscObj *pObj = pSql->pTscObj; + SSqlCmd * pCmd = &pSql->cmd; + STscObj * pObj = pSql->pTscObj; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + pMsg = pCmd->payload + tsRpcHeadSize; pStart = pMsg; @@ -1517,7 +1890,7 @@ static int tscBuildAcctMsgImpl(SSqlObj *pSql) { pMsg += sizeof(SMgmtHead); pAlterMsg = (SCreateAcctMsg *)pMsg; - strcpy(pAlterMsg->user, pCmd->name); + strcpy(pAlterMsg->user, pMeterMetaInfo->name); strcpy(pAlterMsg->pass, pCmd->payload); pMsg += sizeof(SCreateAcctMsg); @@ -1555,8 +1928,10 @@ int tscBuildAlterUserMsg(SSqlObj *pSql) { char * pMsg, *pStart; int msgLen = 0; - SSqlCmd *pCmd = &pSql->cmd; - STscObj *pObj = pSql->pTscObj; + SSqlCmd * pCmd = &pSql->cmd; + STscObj * pObj = pSql->pTscObj; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + pMsg = pCmd->payload + tsRpcHeadSize; pStart = pMsg; @@ -1565,7 +1940,7 @@ int tscBuildAlterUserMsg(SSqlObj *pSql) { pMsg += sizeof(SMgmtHead); pAlterMsg = (SCreateUserMsg *)pMsg; - strcpy(pAlterMsg->user, pCmd->name); + strcpy(pAlterMsg->user, pMeterMetaInfo->name); strcpy(pAlterMsg->pass, pCmd->payload); pAlterMsg->flag = pCmd->order.order; pAlterMsg->privilege = (char)pCmd->count; @@ -1584,8 +1959,10 @@ int tscBuildCfgDnodeMsg(SSqlObj *pSql) { char * pMsg, *pStart; int msgLen = 0; - SSqlCmd *pCmd = &pSql->cmd; - STscObj *pObj = pSql->pTscObj; + SSqlCmd * pCmd = &pSql->cmd; + STscObj * pObj = pSql->pTscObj; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + pMsg = pCmd->payload + tsRpcHeadSize; pStart = pMsg; @@ -1594,7 +1971,7 @@ int tscBuildCfgDnodeMsg(SSqlObj *pSql) { pMsg += sizeof(SMgmtHead); pCfg = (SCfgMsg *)pMsg; - strcpy(pCfg->ip, pCmd->name); + strcpy(pCfg->ip, pMeterMetaInfo->name); strcpy(pCfg->config, pCmd->payload); pMsg += sizeof(SCfgMsg); @@ -1611,8 +1988,10 @@ int tscBuildDropDbMsg(SSqlObj *pSql) { char * pMsg, *pStart; int msgLen = 0; - SSqlCmd *pCmd = &pSql->cmd; - STscObj *pObj = pSql->pTscObj; + SSqlCmd * pCmd = &pSql->cmd; + STscObj * pObj = pSql->pTscObj; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + pMsg = pCmd->payload + tsRpcHeadSize; pStart = pMsg; @@ -1621,7 +2000,7 @@ int tscBuildDropDbMsg(SSqlObj *pSql) { pMsg += sizeof(SMgmtHead); pDropDbMsg = (SDropDbMsg *)pMsg; - strcpy(pDropDbMsg->db, pCmd->name); + strcpy(pDropDbMsg->db, pMeterMetaInfo->name); pDropDbMsg->ignoreNotExists = htons(pCmd->existsCheck ? 1 : 0); @@ -1639,8 +2018,10 @@ int tscBuildDropUserMsg(SSqlObj *pSql) { char * pMsg, *pStart; int msgLen = 0; - SSqlCmd *pCmd = &pSql->cmd; - STscObj *pObj = pSql->pTscObj; + SSqlCmd * pCmd = &pSql->cmd; + STscObj * pObj = pSql->pTscObj; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + pMsg = pCmd->payload + tsRpcHeadSize; pStart = pMsg; @@ -1649,7 +2030,7 @@ int tscBuildDropUserMsg(SSqlObj *pSql) { pMsg += sizeof(SMgmtHead); pDropMsg = (SDropUserMsg *)pMsg; - strcpy(pDropMsg->user, pCmd->name); + strcpy(pDropMsg->user, pMeterMetaInfo->name); pMsg += sizeof(SDropUserMsg); @@ -1665,8 +2046,10 @@ int tscBuildDropAcctMsg(SSqlObj *pSql) { char * pMsg, *pStart; int msgLen = 0; - SSqlCmd *pCmd = &pSql->cmd; - STscObj *pObj = pSql->pTscObj; + SSqlCmd * pCmd = &pSql->cmd; + STscObj * pObj = pSql->pTscObj; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + pMsg = pCmd->payload + tsRpcHeadSize; pStart = pMsg; @@ -1675,7 +2058,7 @@ int tscBuildDropAcctMsg(SSqlObj *pSql) { pMsg += sizeof(SMgmtHead); pDropMsg = (SDropAcctMsg *)pMsg; - strcpy(pDropMsg->user, pCmd->name); + strcpy(pDropMsg->user, pMeterMetaInfo->name); pMsg += sizeof(SDropAcctMsg); @@ -1691,8 +2074,10 @@ int tscBuildUseDbMsg(SSqlObj *pSql) { char * pMsg, *pStart; int msgLen = 0; - SSqlCmd *pCmd = &pSql->cmd; - STscObj *pObj = pSql->pTscObj; + SSqlCmd * pCmd = &pSql->cmd; + STscObj * pObj = pSql->pTscObj; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + pMsg = pCmd->payload + tsRpcHeadSize; pStart = pMsg; @@ -1701,7 +2086,7 @@ int tscBuildUseDbMsg(SSqlObj *pSql) { pMsg += sizeof(SMgmtHead); pUseDbMsg = (SUseDbMsg *)pMsg; - strcpy(pUseDbMsg->db, pCmd->name); + strcpy(pUseDbMsg->db, pMeterMetaInfo->name); pMsg += sizeof(SUseDbMsg); @@ -1725,14 +2110,21 @@ int tscBuildShowMsg(SSqlObj *pSql) { memcpy(payload, pCmd->payload, pCmd->payloadLen); int32_t size = minMsgSize() + sizeof(SMgmtHead) + sizeof(SShowTableMsg) + pCmd->payloadLen + TSDB_EXTRA_PAYLOAD_SIZE; - tscAllocPayloadWithSize(pCmd, size); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + tscError("%p failed to malloc for show msg", pSql); + return -1; + } pMsg = pCmd->payload + tsRpcHeadSize; pStart = pMsg; SMgmtHead *pMgmt = (SMgmtHead *)pMsg; - if (pCmd->tagCond.len > 0) { - strcpy(pMgmt->db, pCmd->tagCond.pData); + + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + size_t nameLen = strlen(pMeterMetaInfo->name); + + if (nameLen > 0) { + strcpy(pMgmt->db, pMeterMetaInfo->name); } else { strcpy(pMgmt->db, pObj->db); } @@ -1740,10 +2132,9 @@ int tscBuildShowMsg(SSqlObj *pSql) { pMsg += sizeof(SMgmtHead); pShowMsg = (SShowMsg *)pMsg; - pShowMsg->type = pCmd->type; + pShowMsg->type = pCmd->showType; - if ((pShowMsg->type == TSDB_MGMT_TABLE_TABLE || pShowMsg->type == TSDB_MGMT_TABLE_METRIC) && - pCmd->payloadLen != 0) { + if ((pShowMsg->type == TSDB_MGMT_TABLE_TABLE || pShowMsg->type == TSDB_MGMT_TABLE_METRIC) && pCmd->payloadLen != 0) { // only show tables support wildcard query pShowMsg->payloadLen = htons(pCmd->payloadLen); memcpy(pShowMsg->payload, payload, pCmd->payloadLen); @@ -1871,23 +2262,27 @@ int tscBuildCreateTableMsg(SSqlObj *pSql) { // STagData is in binary format, strncpy is not available memcpy(tmpData, pSql->cmd.payload, pSql->cmd.allocSize); - SSqlCmd *pCmd = &pSql->cmd; + SSqlCmd * pCmd = &pSql->cmd; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); // Reallocate the payload size size = tscEstimateCreateTableMsgLength(pSql); - tscAllocPayloadWithSize(pCmd, size); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + tscError("%p failed to malloc for create table msg", pSql); + return -1; + } pMsg = pCmd->payload + tsRpcHeadSize; pStart = pMsg; SMgmtHead *pMgmt = (SMgmtHead *)pMsg; // use dbinfo from meterid without modifying current db info - tscGetDBInfoFromMeterId(pSql->cmd.name, pMgmt->db); + tscGetDBInfoFromMeterId(pMeterMetaInfo->name, pMgmt->db); pMsg += sizeof(SMgmtHead); pCreateTableMsg = (SCreateTableMsg *)pMsg; - strcpy(pCreateTableMsg->meterId, pCmd->name); + strcpy(pCreateTableMsg->meterId, pMeterMetaInfo->name); pCreateTableMsg->igExists = pCmd->existsCheck ? 1 : 0; pCreateTableMsg->numOfColumns = htons(pCmd->numOfCols); @@ -1926,7 +2321,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql) { } tfree(tmpData); - tscClearFieldInfo(pCmd); + tscClearFieldInfo(&pCmd->fieldsInfo); msgLen = pMsg - pStart; pCmd->payloadLen = msgLen; @@ -1947,24 +2342,28 @@ int tscBuildAlterTableMsg(SSqlObj *pSql) { int msgLen = 0; int size = 0; - SSqlCmd *pCmd = &pSql->cmd; + SSqlCmd * pCmd = &pSql->cmd; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); - char buf[TSDB_MAX_TAGS_LEN] = {0}; - int32_t len = (TSDB_MAX_TAGS_LEN < pCmd->allocSize)? TSDB_MAX_TAGS_LEN:pCmd->allocSize; + char buf[TSDB_MAX_TAGS_LEN] = {0}; + int32_t len = (TSDB_MAX_TAGS_LEN < pCmd->allocSize) ? TSDB_MAX_TAGS_LEN : pCmd->allocSize; memcpy(buf, pCmd->payload, len); size = tscEstimateAlterTableMsgLength(pCmd); - tscAllocPayloadWithSize(pCmd, size); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + tscError("%p failed to malloc for alter table msg", pSql); + return -1; + } pMsg = pCmd->payload + tsRpcHeadSize; pStart = pMsg; SMgmtHead *pMgmt = (SMgmtHead *)pMsg; - tscGetDBInfoFromMeterId(pCmd->name, pMgmt->db); + tscGetDBInfoFromMeterId(pMeterMetaInfo->name, pMgmt->db); pMsg += sizeof(SMgmtHead); pAlterTableMsg = (SAlterTableMsg *)pMsg; - strcpy(pAlterTableMsg->meterId, pCmd->name); + strcpy(pAlterTableMsg->meterId, pMeterMetaInfo->name); pAlterTableMsg->type = htons(pCmd->count); pAlterTableMsg->numOfCols = htons(pCmd->numOfCols); memcpy(pAlterTableMsg->tagVal, buf, TSDB_MAX_TAGS_LEN); @@ -1994,8 +2393,10 @@ int tscAlterDbMsg(SSqlObj *pSql) { char * pMsg, *pStart; int msgLen = 0; - SSqlCmd *pCmd = &pSql->cmd; - STscObj *pObj = pSql->pTscObj; + SSqlCmd * pCmd = &pSql->cmd; + STscObj * pObj = pSql->pTscObj; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + pStart = pCmd->payload + tsRpcHeadSize; pMsg = pStart; @@ -2004,11 +2405,7 @@ int tscAlterDbMsg(SSqlObj *pSql) { pMsg += sizeof(SMgmtHead); pAlterDbMsg = (SAlterDbMsg *)pMsg; - strcpy(pAlterDbMsg->db, pCmd->name); - - pAlterDbMsg->replications = pCmd->defaultVal[0]; - pAlterDbMsg->daysPerFile = htonl(pCmd->defaultVal[1]); - pAlterDbMsg->daysToKeep = htonl(pCmd->defaultVal[2]); + strcpy(pAlterDbMsg->db, pMeterMetaInfo->name); pMsg += sizeof(SAlterDbMsg); @@ -2024,16 +2421,18 @@ int tscBuildDropTableMsg(SSqlObj *pSql) { char * pMsg, *pStart; int msgLen = 0; - SSqlCmd *pCmd = &pSql->cmd; + SSqlCmd * pCmd = &pSql->cmd; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + pMsg = pCmd->payload + tsRpcHeadSize; pStart = pMsg; SMgmtHead *pMgmt = (SMgmtHead *)pMsg; - tscGetDBInfoFromMeterId(pCmd->name, pMgmt->db); + tscGetDBInfoFromMeterId(pMeterMetaInfo->name, pMgmt->db); pMsg += sizeof(SMgmtHead); pDropTableMsg = (SDropTableMsg *)pMsg; - strcpy(pDropTableMsg->meterId, pCmd->name); + strcpy(pDropTableMsg->meterId, pMeterMetaInfo->name); pDropTableMsg->igNotExists = pCmd->existsCheck ? 1 : 0; pMsg += sizeof(SDropTableMsg); @@ -2055,17 +2454,23 @@ int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql) { pStart = pMsg; SMgmtHead *pMgmt = (SMgmtHead *)pMsg; - if (pCmd->tagCond.len > 0) { - strcpy(pMgmt->db, pCmd->tagCond.pData); + + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + size_t nameLen = strlen(pMeterMetaInfo->name); + + if (nameLen > 0) { + strcpy(pMgmt->db, pMeterMetaInfo->name); } else { strcpy(pMgmt->db, pObj->db); } + pMsg += sizeof(SMgmtHead); *((uint64_t *)pMsg) = pSql->res.qhandle; - pMsg += 8; - *pMsg = pCmd->type; - pMsg += 1; + pMsg += sizeof(pSql->res.qhandle); + + *pMsg = htons(pCmd->type); + pMsg += sizeof(pCmd->type); msgLen = pMsg - pStart; pCmd->payloadLen = msgLen; @@ -2116,8 +2521,7 @@ static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { pRes->row = 0; } else { - pRes->numOfRows = 0; - pRes->row = 0; + tscResetForNextRetrieve(pRes); } uint8_t code = pSql->res.code; @@ -2133,17 +2537,21 @@ static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { } int tscProcessDescribeTableRsp(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - int32_t numOfRes = pCmd->pMeterMeta->numOfColumns + pCmd->pMeterMeta->numOfTags; + SSqlCmd * pCmd = &pSql->cmd; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + + int32_t numOfRes = pMeterMetaInfo->pMeterMeta->numOfColumns + pMeterMetaInfo->pMeterMeta->numOfTags; return tscLocalResultCommonBuilder(pSql, numOfRes); } int tscProcessTagRetrieveRsp(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - int32_t numOfRes = 0; - if (tscSqlExprGet(pCmd, 0)->sqlFuncId == TSDB_FUNC_TAGPRJ) { - numOfRes = pCmd->pMetricMeta->numOfMeters; + SSqlCmd * pCmd = &pSql->cmd; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + + int32_t numOfRes = 0; + if (tscSqlExprGet(pCmd, 0)->functionId == TSDB_FUNC_TAGPRJ) { + numOfRes = pMeterMetaInfo->pMetricMeta->numOfMeters; } else { numOfRes = 1; // for count function, there is only one output. } @@ -2174,9 +2582,7 @@ int tscProcessRetrieveMetricRsp(SSqlObj *pSql) { return code; } -int tscProcessEmptyResultRsp(SSqlObj *pSql) { - return tscLocalResultCommonBuilder(pSql, 0); -} +int tscProcessEmptyResultRsp(SSqlObj *pSql) { return tscLocalResultCommonBuilder(pSql, 0); } int tscBuildConnectMsg(SSqlObj *pSql) { SConnectMsg *pConnect; @@ -2212,21 +2618,24 @@ int tscBuildMeterMetaMsg(SSqlObj *pSql) { char *tmpData = 0; if (pSql->cmd.allocSize > 0) { tmpData = calloc(1, pSql->cmd.allocSize); + if (NULL == tmpData) return -1; // STagData is in binary format, strncpy is not available memcpy(tmpData, pSql->cmd.payload, pSql->cmd.allocSize); } - SSqlCmd *pCmd = &pSql->cmd; + SSqlCmd * pCmd = &pSql->cmd; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + pMsg = pCmd->payload + tsRpcHeadSize; pStart = pMsg; SMgmtHead *pMgmt = (SMgmtHead *)pMsg; - tscGetDBInfoFromMeterId(pCmd->name, pMgmt->db); + tscGetDBInfoFromMeterId(pMeterMetaInfo->name, pMgmt->db); pMsg += sizeof(SMgmtHead); pInfoMsg = (SMeterInfoMsg *)pMsg; - strcpy(pInfoMsg->meterId, pCmd->name); + strcpy(pInfoMsg->meterId, pMeterMetaInfo->name); pInfoMsg->createFlag = htons((uint16_t)pCmd->defaultVal[0]); pMsg += sizeof(SMeterInfoMsg); @@ -2245,78 +2654,196 @@ int tscBuildMeterMetaMsg(SSqlObj *pSql) { return msgLen; } +/** + * multi meter meta req pkg format: + * | SMgmtHead | SMultiMeterInfoMsg | meterId0 | meterId1 | meterId2 | ...... + * no used 4B + **/ +int tscBuildMultiMeterMetaMsg(SSqlObj *pSql) { + SSqlCmd *pCmd = &pSql->cmd; + + // copy payload content to temp buff + char *tmpData = 0; + if (pCmd->payloadLen > 0) { + tmpData = calloc(1, pCmd->payloadLen + 1); + if (NULL == tmpData) return -1; + memcpy(tmpData, pCmd->payload, pCmd->payloadLen); + } + + // fill head info + SMgmtHead *pMgmt = (SMgmtHead *)(pCmd->payload + tsRpcHeadSize); + memset(pMgmt->db, 0, TSDB_METER_ID_LEN); // server don't need the db + + SMultiMeterInfoMsg *pInfoMsg = (SMultiMeterInfoMsg *)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead)); + pInfoMsg->numOfMeters = htonl((int32_t)pCmd->count); + + if (pCmd->payloadLen > 0) { + memcpy(pInfoMsg->meterId, tmpData, pCmd->payloadLen); + } + + tfree(tmpData); + + pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SMultiMeterInfoMsg); + pCmd->msgType = TSDB_MSG_TYPE_MULTI_METERINFO; + + assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize); + + tscTrace("%p build load multi-metermeta msg completed, numOfMeters:%d, msg size:%d", pSql, pCmd->count, + pCmd->payloadLen); + + return pCmd->payloadLen; +} + static int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) { const int32_t defaultSize = minMsgSize() + sizeof(SMetricMetaMsg) + sizeof(SMgmtHead) + sizeof(int16_t) * TSDB_MAX_TAGS; - int32_t tagLen = pCmd->tagCond.len * TSDB_NCHAR_SIZE; - if (tagLen + defaultSize > TSDB_DEFAULT_PAYLOAD_SIZE) { - return tagLen + defaultSize; - } else { - return TSDB_DEFAULT_PAYLOAD_SIZE; + int32_t n = 0; + for (int32_t i = 0; i < pCmd->tagCond.numOfTagCond; ++i) { + n += pCmd->tagCond.cond[i].cond.n; } + + int32_t tagLen = n * TSDB_NCHAR_SIZE + pCmd->tagCond.tbnameCond.cond.n * TSDB_NCHAR_SIZE; + int32_t joinCondLen = (TSDB_METER_ID_LEN + sizeof(int16_t)) * 2; + int32_t elemSize = sizeof(SMetricMetaElemMsg) * pCmd->numOfTables; + + int32_t len = tagLen + joinCondLen + elemSize + defaultSize; + + return MAX(len, TSDB_DEFAULT_PAYLOAD_SIZE); } int tscBuildMetricMetaMsg(SSqlObj *pSql) { SMetricMetaMsg *pMetaMsg; char * pMsg, *pStart; int msgLen = 0; + int tableIndex = 0; - SSqlCmd *pCmd = &pSql->cmd; + SSqlCmd * pCmd = &pSql->cmd; + STagCond *pTagCond = &pCmd->tagCond; + + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, tableIndex); int32_t size = tscEstimateMetricMetaMsgSize(pCmd); - tscAllocPayloadWithSize(pCmd, size); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + tscError("%p failed to malloc for metric meter msg", pSql); + return -1; + } pStart = pCmd->payload + tsRpcHeadSize; pMsg = pStart; SMgmtHead *pMgmt = (SMgmtHead *)pMsg; - tscGetDBInfoFromMeterId(pCmd->name, pMgmt->db); + tscGetDBInfoFromMeterId(pMeterMetaInfo->name, pMgmt->db); pMsg += sizeof(SMgmtHead); pMetaMsg = (SMetricMetaMsg *)pMsg; - strcpy(pMetaMsg->meterId, pCmd->name); + pMetaMsg->numOfMeters = htonl(pCmd->numOfTables); + + pMsg += sizeof(SMetricMetaMsg); + + int32_t offset = pMsg - (char *)pMetaMsg; + pMetaMsg->join = htonl(offset); + + // todo refactor + pMetaMsg->joinCondLen = htonl((TSDB_METER_ID_LEN + sizeof(int16_t)) * 2); - pMetaMsg->type = htons(pCmd->tagCond.type); - pMetaMsg->condLength = htonl(pCmd->tagCond.len); + memcpy(pMsg, pTagCond->joinInfo.left.meterId, TSDB_METER_ID_LEN); + pMsg += TSDB_METER_ID_LEN; - if (pCmd->tagCond.len > 0) { - /* convert to unicode before sending to mnode for metric query */ - bool ret = taosMbsToUcs4(tsGetMetricQueryCondPos(&pCmd->tagCond), pCmd->tagCond.len, (char *)pMetaMsg->tags, - pCmd->tagCond.len * TSDB_NCHAR_SIZE); - if (!ret) { - tscError("%p mbs to ucs4 failed:%s", pSql, tsGetMetricQueryCondPos(&pCmd->tagCond)); - return 0; + *(int16_t *)pMsg = pTagCond->joinInfo.left.tagCol; + pMsg += sizeof(int16_t); + + memcpy(pMsg, pTagCond->joinInfo.right.meterId, TSDB_METER_ID_LEN); + pMsg += TSDB_METER_ID_LEN; + + *(int16_t *)pMsg = pTagCond->joinInfo.right.tagCol; + pMsg += sizeof(int16_t); + + for (int32_t i = 0; i < pCmd->numOfTables; ++i) { + pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, i); + uint64_t uid = pMeterMetaInfo->pMeterMeta->uid; + + offset = pMsg - (char *)pMetaMsg; + pMetaMsg->metaElem[i] = htonl(offset); + + SMetricMetaElemMsg *pElem = (SMetricMetaElemMsg *)pMsg; + pMsg += sizeof(SMetricMetaElemMsg); + + // convert to unicode before sending to mnode for metric query + int32_t condLen = 0; + if (pTagCond->numOfTagCond > 0) { + SCond *pCond = tsGetMetricQueryCondPos(pTagCond, uid); + if (pCond != NULL) { + condLen = pCond->cond.n + 1; + bool ret = taosMbsToUcs4(pCond->cond.z, pCond->cond.n, pMsg, pCond->cond.n * TSDB_NCHAR_SIZE); + if (!ret) { + tscError("%p mbs to ucs4 failed:%s", pSql, tsGetMetricQueryCondPos(pTagCond, uid)); + return 0; + } + } } - } - pMsg += sizeof(SMetricMetaMsg); - pMsg += pCmd->tagCond.len * TSDB_NCHAR_SIZE; + pElem->condLen = htonl(condLen); - SSqlGroupbyExpr *pGroupby = &pCmd->groupbyExpr; + offset = pMsg - (char *)pMetaMsg; + pElem->cond = htonl(offset); + pMsg += condLen * TSDB_NCHAR_SIZE; - pMetaMsg->numOfTags = htons(pCmd->numOfReqTags); - pMetaMsg->numOfGroupbyCols = htons(pGroupby->numOfGroupbyCols); + pElem->rel = htons(pTagCond->relType); + if (pTagCond->tbnameCond.uid == uid) { + offset = pMsg - (char *)pMetaMsg; - for (int32_t j = 0; j < pCmd->numOfReqTags; ++j) { - pMetaMsg->tagCols[j] = htons(pCmd->tagColumnIndex[j]); - } + pElem->tableCond = htonl(offset); + pElem->tableCondLen = htonl(pTagCond->tbnameCond.cond.n); + + memcpy(pMsg, pTagCond->tbnameCond.cond.z, pTagCond->tbnameCond.cond.n); + pMsg += pTagCond->tbnameCond.cond.n; + } + + SSqlGroupbyExpr *pGroupby = &pCmd->groupbyExpr; + + if (pGroupby->tableIndex != i) { + pElem->orderType = 0; + pElem->orderIndex = 0; + pElem->numOfGroupCols = 0; + } else { + pElem->numOfGroupCols = htons(pGroupby->numOfGroupCols); + for (int32_t j = 0; j < pMeterMetaInfo->numOfTags; ++j) { + pElem->tagCols[j] = htons(pMeterMetaInfo->tagColumnIndex[j]); + } + + if (pGroupby->numOfGroupCols != 0) { + pElem->orderIndex = htons(pGroupby->orderIndex); + pElem->orderType = htons(pGroupby->orderType); + offset = pMsg - (char *)pMetaMsg; + + pElem->groupbyTagColumnList = htonl(offset); + for (int32_t j = 0; j < pCmd->groupbyExpr.numOfGroupCols; ++j) { + SColIndexEx *pCol = &pCmd->groupbyExpr.columnInfo[j]; + + *((int16_t *)pMsg) = pCol->colId; + pMsg += sizeof(pCol->colId); - if (pGroupby->numOfGroupbyCols != 0) { - pMetaMsg->orderIndex = htons(pGroupby->orderIdx); - pMetaMsg->orderType = htons(pGroupby->orderType); + *((int16_t *)pMsg) += pCol->colIdx; + pMsg += sizeof(pCol->colIdx); - for (int32_t j = 0; j < pCmd->groupbyExpr.numOfGroupbyCols; ++j) { - *((int16_t *)pMsg) = htons(pGroupby->tagIndex[j]); - pMsg += sizeof(pCmd->groupbyExpr.tagIndex[j]); + *((int16_t *)pMsg) += pCol->flag; + pMsg += sizeof(pCol->flag); + } + } } + + strcpy(pElem->meterId, pMeterMetaInfo->name); + pElem->numOfTags = htons(pMeterMetaInfo->numOfTags); + + int16_t len = pMsg - (char *)pElem; + pElem->elemLen = htons(len); // redundant data for integrate check } msgLen = pMsg - pStart; pCmd->payloadLen = msgLen; pCmd->msgType = TSDB_MSG_TYPE_METRIC_META; - assert(msgLen + minMsgSize() <= size); return msgLen; } @@ -2355,7 +2882,10 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql) { pthread_mutex_lock(&pObj->mutex); size = tscEstimateBuildHeartBeatMsgLength(pSql); - tscAllocPayloadWithSize(pCmd, size); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + tscError("%p failed to malloc for heartbeat msg", pSql); + return -1; + } pMsg = pCmd->payload + tsRpcHeadSize; pStart = pMsg; @@ -2414,7 +2944,7 @@ int tscProcessMeterMetaRsp(SSqlObj *pSql) { pMeta = (SMeterMeta *)rsp; pMeta->sid = htonl(pMeta->sid); - pMeta->sversion = htonl(pMeta->sversion); + pMeta->sversion = htons(pMeta->sversion); pMeta->vgid = htonl(pMeta->vgid); pMeta->uid = htobe64(pMeta->uid); @@ -2424,9 +2954,6 @@ int tscProcessMeterMetaRsp(SSqlObj *pSql) { } pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->numOfTags = htons(pMeta->numOfTags); - pMeta->precision = htons(pMeta->precision); - pMeta->meterType = htons(pMeta->meterType); if (pMeta->numOfTags > TSDB_MAX_TAGS || pMeta->numOfTags < 0) { tscError("invalid tag value count:%d", pMeta->numOfTags); @@ -2466,13 +2993,12 @@ int tscProcessMeterMetaRsp(SSqlObj *pSql) { rsp += numOfTotalCols * sizeof(SSchema); int32_t tagLen = 0; - SSchema *pTagsSchema = tsGetSchemaColIdx(pMeta, pMeta->numOfColumns); + SSchema *pTagsSchema = tsGetTagSchema(pMeta); if (pMeta->meterType == TSDB_METER_MTABLE) { for (int32_t i = 0; i < pMeta->numOfTags; ++i) { tagLen += pTagsSchema[i].bytes; } - pMeta->tags = sizeof(SMeterMeta) + numOfTotalCols * sizeof(SSchema); } rsp += tagLen; @@ -2482,19 +3008,133 @@ int tscProcessMeterMetaRsp(SSqlObj *pSql) { pMeta->index = 0; // todo add one more function: taosAddDataIfNotExists(); - taosRemoveDataFromCache(tscCacheHandle, (void **)&(pSql->cmd.pMeterMeta), false); + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0); + taosRemoveDataFromCache(tscCacheHandle, (void **)&(pMeterMetaInfo->pMeterMeta), false); - pSql->cmd.pMeterMeta = - (SMeterMeta *)taosAddDataIntoCache(tscCacheHandle, pSql->cmd.name, (char *)pMeta, size, tsMeterMetaKeepTimer); - if (pSql->cmd.pMeterMeta == NULL) return 0; + pMeterMetaInfo->pMeterMeta = (SMeterMeta *)taosAddDataIntoCache(tscCacheHandle, pMeterMetaInfo->name, (char *)pMeta, + size, tsMeterMetaKeepTimer); + if (pMeterMetaInfo->pMeterMeta == NULL) return 0; return TSDB_CODE_OTHERS; } +/** + * multi meter meta rsp pkg format: + * | STaosRsp | ieType | SMultiMeterInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 + * |...... 1B 1B 4B + **/ +int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { + SSchema *pSchema; + uint8_t ieType; + int32_t totalNum; + int32_t i; + + char *rsp = pSql->res.pRsp; + + ieType = *rsp; + if (ieType != TSDB_IE_TYPE_META) { + tscError("invalid ie type:%d", ieType); + pSql->res.code = TSDB_CODE_INVALID_IE; + pSql->res.numOfTotal = 0; + return TSDB_CODE_OTHERS; + } + + rsp++; + + SMultiMeterInfoMsg *pInfo = (SMultiMeterInfoMsg *)rsp; + totalNum = htonl(pInfo->numOfMeters); + rsp += sizeof(SMultiMeterInfoMsg); + + for (i = 0; i < totalNum; i++) { + SMultiMeterMeta *pMultiMeta = (SMultiMeterMeta *)rsp; + SMeterMeta * pMeta = &pMultiMeta->meta; + + pMeta->sid = htonl(pMeta->sid); + pMeta->sversion = htons(pMeta->sversion); + pMeta->vgid = htonl(pMeta->vgid); + pMeta->uid = htobe64(pMeta->uid); + + if (pMeta->sid <= 0 || pMeta->vgid < 0) { + tscError("invalid meter vgid:%d, sid%d", pMeta->vgid, pMeta->sid); + pSql->res.code = TSDB_CODE_INVALID_VALUE; + pSql->res.numOfTotal = i; + return TSDB_CODE_OTHERS; + } + + pMeta->numOfColumns = htons(pMeta->numOfColumns); + + if (pMeta->numOfTags > TSDB_MAX_TAGS || pMeta->numOfTags < 0) { + tscError("invalid tag value count:%d", pMeta->numOfTags); + pSql->res.code = TSDB_CODE_INVALID_VALUE; + pSql->res.numOfTotal = i; + return TSDB_CODE_OTHERS; + } + + if (pMeta->numOfTags > TSDB_MAX_TAGS || pMeta->numOfTags < 0) { + tscError("invalid numOfTags:%d", pMeta->numOfTags); + pSql->res.code = TSDB_CODE_INVALID_VALUE; + pSql->res.numOfTotal = i; + return TSDB_CODE_OTHERS; + } + + if (pMeta->numOfColumns > TSDB_MAX_COLUMNS || pMeta->numOfColumns < 0) { + tscError("invalid numOfColumns:%d", pMeta->numOfColumns); + pSql->res.code = TSDB_CODE_INVALID_VALUE; + pSql->res.numOfTotal = i; + return TSDB_CODE_OTHERS; + } + + for (int j = 0; j < TSDB_VNODES_SUPPORT; ++j) { + pMeta->vpeerDesc[j].vnode = htonl(pMeta->vpeerDesc[j].vnode); + } + + pMeta->rowSize = 0; + rsp += sizeof(SMultiMeterMeta); + pSchema = (SSchema *)rsp; + + int32_t numOfTotalCols = pMeta->numOfColumns + pMeta->numOfTags; + for (int j = 0; j < numOfTotalCols; ++j) { + pSchema->bytes = htons(pSchema->bytes); + pSchema->colId = htons(pSchema->colId); + + // ignore the tags length + if (j < pMeta->numOfColumns) { + pMeta->rowSize += pSchema->bytes; + } + pSchema++; + } + + rsp += numOfTotalCols * sizeof(SSchema); + + int32_t tagLen = 0; + SSchema *pTagsSchema = tsGetTagSchema(pMeta); + + if (pMeta->meterType == TSDB_METER_MTABLE) { + for (int32_t j = 0; j < pMeta->numOfTags; ++j) { + tagLen += pTagsSchema[j].bytes; + } + } + + rsp += tagLen; + int32_t size = (int32_t)(rsp - ((char *)pMeta)); // Consistent with SMeterMeta in cache + + pMeta->index = 0; + (void)taosAddDataIntoCache(tscCacheHandle, pMultiMeta->meterId, (char *)pMeta, size, tsMeterMetaKeepTimer); + } + + pSql->res.code = TSDB_CODE_SUCCESS; + pSql->res.numOfTotal = i; + tscTrace("%p load multi-metermeta resp complete num:%d", pSql, pSql->res.numOfTotal); + return TSDB_CODE_SUCCESS; +} + int tscProcessMetricMetaRsp(SSqlObj *pSql) { SMetricMeta *pMeta; uint8_t ieType; - char * rsp = pSql->res.pRsp; + void ** metricMetaList = NULL; + int32_t * sizes = NULL; + + char *rsp = pSql->res.pRsp; ieType = *rsp; if (ieType != TSDB_IE_TYPE_META) { @@ -2503,62 +3143,108 @@ int tscProcessMetricMetaRsp(SSqlObj *pSql) { } rsp++; - pMeta = (SMetricMeta *)rsp; - size_t size = (size_t)pSql->res.rspLen - 1; - rsp = rsp + sizeof(SMetricMeta); - pMeta->numOfMeters = htonl(pMeta->numOfMeters); - pMeta->numOfVnodes = htonl(pMeta->numOfVnodes); - pMeta->tagLen = htons(pMeta->tagLen); + int32_t num = htons(*(int16_t *)rsp); + rsp += sizeof(int16_t); + + metricMetaList = calloc(1, POINTER_BYTES * num); + sizes = calloc(1, sizeof(int32_t) * num); + + // return with error code + if (metricMetaList == NULL || sizes == NULL) { + tfree(metricMetaList); + tfree(sizes); + pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY; + + return pSql->res.code; + } + + for (int32_t k = 0; k < num; ++k) { + pMeta = (SMetricMeta *)rsp; + + size_t size = (size_t)pSql->res.rspLen - 1; + rsp = rsp + sizeof(SMetricMeta); + + pMeta->numOfMeters = htonl(pMeta->numOfMeters); + pMeta->numOfVnodes = htonl(pMeta->numOfVnodes); + pMeta->tagLen = htons(pMeta->tagLen); + + size += pMeta->numOfVnodes * sizeof(SVnodeSidList *) + pMeta->numOfMeters * sizeof(SMeterSidExtInfo *); - size += pMeta->numOfVnodes * sizeof(SVnodeSidList *) + pMeta->numOfMeters * sizeof(SMeterSidExtInfo *); + char *pStr = calloc(1, size); + if (pStr == NULL) { + pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY; + goto _error_clean; + } - char *pStr = calloc(1, size); + SMetricMeta *pNewMetricMeta = (SMetricMeta *)pStr; + metricMetaList[k] = pNewMetricMeta; - SMetricMeta *pNewMetricMeta = (SMetricMeta *)pStr; - pNewMetricMeta->numOfMeters = pMeta->numOfMeters; - pNewMetricMeta->numOfVnodes = pMeta->numOfVnodes; - pNewMetricMeta->tagLen = pMeta->tagLen; + pNewMetricMeta->numOfMeters = pMeta->numOfMeters; + pNewMetricMeta->numOfVnodes = pMeta->numOfVnodes; + pNewMetricMeta->tagLen = pMeta->tagLen; - pStr = pStr + sizeof(SMetricMeta) + pNewMetricMeta->numOfVnodes * sizeof(SVnodeSidList *); + pStr = pStr + sizeof(SMetricMeta) + pNewMetricMeta->numOfVnodes * sizeof(SVnodeSidList *); - for (int32_t i = 0; i < pMeta->numOfVnodes; ++i) { - SVnodeSidList *pSidLists = (SVnodeSidList *)rsp; - memcpy(pStr, pSidLists, sizeof(SVnodeSidList)); + for (int32_t i = 0; i < pMeta->numOfVnodes; ++i) { + SVnodeSidList *pSidLists = (SVnodeSidList *)rsp; + memcpy(pStr, pSidLists, sizeof(SVnodeSidList)); - pNewMetricMeta->list[i] = pStr - (char *)pNewMetricMeta; // offset value - SVnodeSidList *pLists = (SVnodeSidList *)pStr; + pNewMetricMeta->list[i] = pStr - (char *)pNewMetricMeta; // offset value + SVnodeSidList *pLists = (SVnodeSidList *)pStr; - tscTrace("%p metricmeta:vid:%d,numOfMeters:%d", pSql, i, pLists->numOfSids); + tscTrace("%p metricmeta:vid:%d,numOfMeters:%d", pSql, i, pLists->numOfSids); - pStr += sizeof(SVnodeSidList) + sizeof(SMeterSidExtInfo *) * pSidLists->numOfSids; - rsp += sizeof(SVnodeSidList); + pStr += sizeof(SVnodeSidList) + sizeof(SMeterSidExtInfo *) * pSidLists->numOfSids; + rsp += sizeof(SVnodeSidList); - size_t sidSize = sizeof(SMeterSidExtInfo) + pNewMetricMeta->tagLen; - for (int32_t j = 0; j < pSidLists->numOfSids; ++j) { - pLists->pSidExtInfoList[j] = pStr - (char *)pLists; - memcpy(pStr, rsp, sidSize); + size_t sidSize = sizeof(SMeterSidExtInfo) + pNewMetricMeta->tagLen; + for (int32_t j = 0; j < pSidLists->numOfSids; ++j) { + pLists->pSidExtInfoList[j] = pStr - (char *)pLists; + memcpy(pStr, rsp, sidSize); - rsp += sidSize; - pStr += sidSize; + rsp += sidSize; + pStr += sidSize; + } } + + sizes[k] = pStr - (char *)pNewMetricMeta; } - char name[TSDB_MAX_TAGS_LEN + 1] = {0}; - tscGetMetricMetaCacheKey(&pSql->cmd, name); + for (int32_t i = 0; i < num; ++i) { + char name[TSDB_MAX_TAGS_LEN + 1] = {0}; + + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, i); + tscGetMetricMetaCacheKey(&pSql->cmd, name, pMeterMetaInfo->pMeterMeta->uid); - /* release the used metricmeta */ - taosRemoveDataFromCache(tscCacheHandle, (void **)&(pSql->cmd.pMetricMeta), false); +#ifdef _DEBUG_VIEW + printf("generate the metric key:%s, index:%d\n", name, i); +#endif - pSql->cmd.pMetricMeta = - (SMetricMeta *)taosAddDataIntoCache(tscCacheHandle, name, (char *)pNewMetricMeta, size, tsMetricMetaKeepTimer); - tfree(pNewMetricMeta); + // release the used metricmeta + taosRemoveDataFromCache(tscCacheHandle, (void **)&(pMeterMetaInfo->pMetricMeta), false); - if (pSql->cmd.pMetricMeta == NULL) { - return 0; + pMeterMetaInfo->pMetricMeta = (SMetricMeta *)taosAddDataIntoCache(tscCacheHandle, name, (char *)metricMetaList[i], + sizes[i], tsMetricMetaKeepTimer); + tfree(metricMetaList[i]); + + // failed to put into cache + if (pMeterMetaInfo->pMetricMeta == NULL) { + pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY; + goto _error_clean; + } } - return TSDB_CODE_OTHERS; +_error_clean: + // free allocated resource + for (int32_t i = 0; i < num; ++i) { + tfree(metricMetaList[i]); + } + + free(sizes); + free(metricMetaList); + + return pSql->res.code; } /* @@ -2570,14 +3256,14 @@ int tscProcessShowRsp(SSqlObj *pSql) { SSchema * pSchema; char key[20]; - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; + SSqlRes * pRes = &pSql->res; + SSqlCmd * pCmd = &pSql->cmd; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); pShow = (SShowRspMsg *)pRes->pRsp; pRes->qhandle = pShow->qhandle; - pRes->numOfRows = 0; - pRes->row = 0; + tscResetForNextRetrieve(pRes); pMeta = &(pShow->meterMeta); pMeta->numOfColumns = ntohs(pMeta->numOfColumns); @@ -2589,19 +3275,23 @@ int tscProcessShowRsp(SSqlObj *pSql) { pSchema++; } - key[0] = pCmd->type + 'a'; + key[0] = pCmd->showType + 'a'; strcpy(key + 1, "showlist"); - taosRemoveDataFromCache(tscCacheHandle, (void *)&(pCmd->pMeterMeta), false); + taosRemoveDataFromCache(tscCacheHandle, (void *)&(pMeterMetaInfo->pMeterMeta), false); int32_t size = pMeta->numOfColumns * sizeof(SSchema) + sizeof(SMeterMeta); - pCmd->pMeterMeta = (SMeterMeta *)taosAddDataIntoCache(tscCacheHandle, key, (char *)pMeta, size, tsMeterMetaKeepTimer); + pMeterMetaInfo->pMeterMeta = + (SMeterMeta *)taosAddDataIntoCache(tscCacheHandle, key, (char *)pMeta, size, tsMeterMetaKeepTimer); pCmd->numOfCols = pCmd->fieldsInfo.numOfOutputCols; - SSchema *pMeterSchema = tsGetSchema(pCmd->pMeterMeta); + SSchema *pMeterSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); - tscColumnInfoReserve(pCmd, pMeta->numOfColumns); - for (int i = 0; i < pMeta->numOfColumns; ++i) { - tscColumnInfoInsert(pCmd, i); + tscColumnBaseInfoReserve(&pCmd->colList, pMeta->numOfColumns); + SColumnIndex index = {0}; + + for (int16_t i = 0; i < pMeta->numOfColumns; ++i) { + index.columnIndex = i; + tscColumnBaseInfoInsert(pCmd, &index); tscFieldInfoSetValFromSchema(&pCmd->fieldsInfo, i, &pMeterSchema[i]); } @@ -2620,7 +3310,20 @@ int tscProcessConnectRsp(SSqlObj *pSql) { strcpy(pObj->acctId, pConnect->acctId); // copy acctId from response sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db); strcpy(pObj->db, temp); +#ifdef CLUSTER + SIpList * pIpList; + char *rsp = pRes->pRsp + sizeof(SConnectRsp); + pIpList = (SIpList *)rsp; + tscMgmtIpList.numOfIps = pIpList->numOfIps; + for (int i = 0; i < pIpList->numOfIps; ++i) { + tinet_ntoa(tscMgmtIpList.ipstr[i], pIpList->ip[i]); + tscMgmtIpList.ip[i] = pIpList->ip[i]; + } + + rsp += sizeof(SIpList) + sizeof(int32_t) * pIpList->numOfIps; + tscPrintMgmtIp(); +#endif strcpy(pObj->sversion, pConnect->version); pObj->writeAuth = pConnect->writeAuth; pObj->superAuth = pConnect->superAuth; @@ -2630,8 +3333,10 @@ int tscProcessConnectRsp(SSqlObj *pSql) { } int tscProcessUseDbRsp(SSqlObj *pSql) { - STscObj *pObj = pSql->pTscObj; - strcpy(pObj->db, pSql->cmd.name); + STscObj * pObj = pSql->pTscObj; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0); + + strcpy(pObj->db, pMeterMetaInfo->name); return 0; } @@ -2641,7 +3346,9 @@ int tscProcessDropDbRsp(SSqlObj *UNUSED_PARAM(pSql)) { } int tscProcessDropTableRsp(SSqlObj *pSql) { - SMeterMeta *pMeterMeta = taosGetDataFromCache(tscCacheHandle, pSql->cmd.name); + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0); + + SMeterMeta *pMeterMeta = taosGetDataFromCache(tscCacheHandle, pMeterMetaInfo->name); if (pMeterMeta == NULL) { /* not in cache, abort */ return 0; @@ -2651,38 +3358,39 @@ int tscProcessDropTableRsp(SSqlObj *pSql) { * 1. if a user drops one table, which is the only table in a vnode, remove operation will incur vnode to be removed. * 2. Then, a user creates a new metric followed by a table with identical name of removed table but different schema, * here the table will reside in a new vnode. - * The cached information is expired, however, we may have lost the ref of original meter. So, clear whole cache instead. + * The cached information is expired, however, we may have lost the ref of original meter. So, clear whole cache + * instead. */ - tscTrace("%p force release metermeta after drop table:%s", pSql, pSql->cmd.name); + tscTrace("%p force release metermeta after drop table:%s", pSql, pMeterMetaInfo->name); taosRemoveDataFromCache(tscCacheHandle, (void **)&pMeterMeta, true); - if (pSql->cmd.pMeterMeta) { - taosRemoveDataFromCache(tscCacheHandle, (void **)&(pSql->cmd.pMeterMeta), true); - taosRemoveDataFromCache(tscCacheHandle, (void **)&(pSql->cmd.pMetricMeta), true); + if (pMeterMetaInfo->pMeterMeta) { + taosRemoveDataFromCache(tscCacheHandle, (void **)&(pMeterMetaInfo->pMeterMeta), true); + taosRemoveDataFromCache(tscCacheHandle, (void **)&(pMeterMetaInfo->pMetricMeta), true); } return 0; } int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { - SMeterMeta *pMeterMeta = taosGetDataFromCache(tscCacheHandle, pSql->cmd.name); + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0); + + SMeterMeta *pMeterMeta = taosGetDataFromCache(tscCacheHandle, pMeterMetaInfo->name); if (pMeterMeta == NULL) { /* not in cache, abort */ return 0; } - tscTrace("%p force release metermeta in cache after alter-table: %s", pSql, pSql->cmd.name); + tscTrace("%p force release metermeta in cache after alter-table: %s", pSql, pMeterMetaInfo->name); taosRemoveDataFromCache(tscCacheHandle, (void **)&pMeterMeta, true); - if (pSql->cmd.pMeterMeta) { - bool isMetric = UTIL_METER_IS_METRIC(&pSql->cmd); + if (pMeterMetaInfo->pMeterMeta) { + bool isMetric = UTIL_METER_IS_METRIC(pMeterMetaInfo); - taosRemoveDataFromCache(tscCacheHandle, (void **)&(pSql->cmd.pMeterMeta), true); - taosRemoveDataFromCache(tscCacheHandle, (void **)&(pSql->cmd.pMetricMeta), true); + taosRemoveDataFromCache(tscCacheHandle, (void **)&(pMeterMetaInfo->pMeterMeta), true); + taosRemoveDataFromCache(tscCacheHandle, (void **)&(pMeterMetaInfo->pMetricMeta), true); - if (isMetric) { - // here, the pCmd->pMeterMeta == NULL - // if it is a metric, reset whole query cache - tscTrace("%p reset query cache since table:%s is metric", pSql, pSql->cmd.name); + if (isMetric) { // if it is a metric, reset whole query cache + tscTrace("%p reset query cache since table:%s is stable", pSql, pMeterMetaInfo->name); taosClearDataCache(tscCacheHandle); } } @@ -2699,29 +3407,56 @@ int tscProcessQueryRsp(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; pRes->qhandle = *((uint64_t *)pRes->pRsp); - pRes->numOfRows = 0; - pRes->row = 0; pRes->data = NULL; + tscResetForNextRetrieve(pRes); return 0; } +static void doDecompressPayload(SSqlCmd *pCmd, SSqlRes *pRes, int16_t compressed) { + if (compressed && pRes->numOfRows > 0) { + SRetrieveMeterRsp *pRetrieve = (SRetrieveMeterRsp *)pRes->pRsp; + + int32_t numOfTotalCols = pCmd->fieldsInfo.numOfOutputCols + pCmd->fieldsInfo.numOfHiddenCols; + int32_t rowSize = pCmd->fieldsInfo.pOffset[numOfTotalCols - 1] + pCmd->fieldsInfo.pFields[numOfTotalCols - 1].bytes; + + // TODO handle the OOM problem + char * buf = malloc(rowSize * pRes->numOfRows); + + int32_t payloadSize = pRes->rspLen - 1 - sizeof(SRetrieveMeterRsp); + assert(payloadSize > 0); + + int32_t decompressedSize = tsDecompressString(pRetrieve->data, payloadSize, 1, buf, rowSize * pRes->numOfRows, 0, 0, 0); + assert(decompressedSize == rowSize * pRes->numOfRows); + + pRes->pRsp = realloc(pRes->pRsp, pRes->rspLen - payloadSize + decompressedSize); + memcpy(pRes->pRsp + sizeof(SRetrieveMeterRsp), buf, decompressedSize); + + free(buf); + } + + pRes->data = ((SRetrieveMeterRsp *)pRes->pRsp)->data; +} + int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) { - SSqlRes * pRes = &pSql->res; - SSqlCmd * pCmd = &pSql->cmd; - STscObj * pObj = pSql->pTscObj; + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + STscObj *pObj = pSql->pTscObj; + SRetrieveMeterRsp *pRetrieve = (SRetrieveMeterRsp *)pRes->pRsp; pRes->numOfRows = htonl(pRetrieve->numOfRows); pRes->precision = htons(pRetrieve->precision); pRes->offset = htobe64(pRetrieve->offset); - pRes->data = pRetrieve->data; - pRes->useconds = pRetrieve->useconds; + pRes->useconds = htobe64(pRetrieve->useconds); + pRetrieve->compress = htons(pRetrieve->compress); + + doDecompressPayload(pCmd, pRes, pRetrieve->compress); tscSetResultPointer(pCmd, pRes); pRes->row = 0; - if (pRes->numOfRows == 0 && !(tscProjectionQueryOnMetric(pSql) && pRes->offset > 0)) { + if (pRes->numOfRows == 0 && !(tscProjectionQueryOnMetric(pCmd) && pRes->offset > 0)) { taosAddConnIntoCache(tscConnCache, pSql->thandle, pSql->ip, pSql->vnode, pObj->user); pSql->thandle = NULL; } else { @@ -2746,20 +3481,30 @@ int tscProcessRetrieveRspFromLocal(SSqlObj *pSql) { void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code); -static int32_t tscDoGetMeterMeta(SSqlObj *pSql, char *meterId) { +static int32_t tscDoGetMeterMeta(SSqlObj *pSql, char *meterId, int32_t index) { int32_t code = TSDB_CODE_SUCCESS; - SSqlObj *pNew = malloc(sizeof(SSqlObj)); - memset(pNew, 0, sizeof(SSqlObj)); + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + if (NULL == pNew) { + tscError("%p malloc failed for new sqlobj to get meter meta", pSql); + return TSDB_CODE_CLI_OUT_OF_MEMORY; + } pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; pNew->cmd.command = TSDB_SQL_META; pNew->cmd.payload = NULL; pNew->cmd.allocSize = 0; + pNew->cmd.defaultVal[0] = pSql->cmd.defaultVal[0]; // flag of create table if not exists - tscAllocPayloadWithSize(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE); + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { + tscError("%p malloc failed for payload to get meter meta", pSql); + free(pNew); + return TSDB_CODE_CLI_OUT_OF_MEMORY; + } + + SMeterMetaInfo *pMeterMetaInfo = tscAddEmptyMeterMetaInfo(&pNew->cmd); - strcpy(pNew->cmd.name, meterId); + strcpy(pMeterMetaInfo->name, meterId); memcpy(pNew->cmd.payload, pSql->cmd.payload, TSDB_DEFAULT_PAYLOAD_SIZE); tscTrace("%p new pSqlObj:%p to get meterMeta", pSql, pNew); @@ -2768,13 +3513,14 @@ static int32_t tscDoGetMeterMeta(SSqlObj *pSql, char *meterId) { tsem_init(&pNew->emptyRspSem, 0, 1); code = tscProcessSql(pNew); + SMeterMetaInfo *pInfo = tscGetMeterMetaInfo(&pSql->cmd, index); + + // update cache only on success get metermeta if (code == TSDB_CODE_SUCCESS) { - /* update cache only on success get metermeta */ - assert(pSql->cmd.pMeterMeta == NULL); - pSql->cmd.pMeterMeta = (SMeterMeta *)taosGetDataFromCache(tscCacheHandle, meterId); + pInfo->pMeterMeta = (SMeterMeta *)taosGetDataFromCache(tscCacheHandle, meterId); } - tscTrace("%p get meter meta complete, code:%d, pMeterMeta:%p", pSql, code, pSql->cmd.pMeterMeta); + tscTrace("%p get meter meta complete, code:%d, pMeterMeta:%p", pSql, code, pInfo->pMeterMeta); tscFreeSqlObj(pNew); } else { @@ -2791,15 +3537,20 @@ static int32_t tscDoGetMeterMeta(SSqlObj *pSql, char *meterId) { return code; } -int tscGetMeterMeta(SSqlObj *pSql, char *meterId) { - SSqlCmd *pCmd = &pSql->cmd; +int tscGetMeterMeta(SSqlObj *pSql, char *meterId, int32_t index) { + SSqlCmd * pCmd = &pSql->cmd; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index); + + // if the SSqlCmd owns a metermeta, release it first + taosRemoveDataFromCache(tscCacheHandle, (void **)&(pMeterMetaInfo->pMeterMeta), false); + pMeterMetaInfo->pMeterMeta = (SMeterMeta *)taosGetDataFromCache(tscCacheHandle, meterId); + + if (pMeterMetaInfo->pMeterMeta != NULL) { + SMeterMeta *pMeterMeta = pMeterMetaInfo->pMeterMeta; + + tscTrace("%p retrieve meterMeta from cache, the number of columns:%d, numOfTags:%d", pSql, pMeterMeta->numOfColumns, + pMeterMeta->numOfTags); - /* if the SSqlCmd owns a metermeta, release it first */ - taosRemoveDataFromCache(tscCacheHandle, (void **)&(pCmd->pMeterMeta), false); - pCmd->pMeterMeta = (SMeterMeta *)taosGetDataFromCache(tscCacheHandle, meterId); - if (pCmd->pMeterMeta != NULL) { - tscTrace("%p the number of columns:%d, numOfTags:%d, addr:%p", pSql, pCmd->pMeterMeta->numOfColumns, - pCmd->pMeterMeta->numOfTags, pCmd->pMeterMeta); return TSDB_CODE_SUCCESS; } @@ -2808,19 +3559,20 @@ int tscGetMeterMeta(SSqlObj *pSql, char *meterId) { * because in metermeta callback function, the tscParse function will generate the submit data blocks */ if (pSql->fp != NULL && pSql->pStream == NULL) { - tscfreeSqlCmdData(pCmd); + tscFreeSqlCmdData(pCmd); } - return tscDoGetMeterMeta(pSql, meterId); + return tscDoGetMeterMeta(pSql, meterId, index); } int tscGetMeterMetaEx(SSqlObj *pSql, char *meterId, bool createIfNotExists) { pSql->cmd.defaultVal[0] = createIfNotExists ? 1 : 0; - return tscGetMeterMeta(pSql, meterId); + return tscGetMeterMeta(pSql, meterId, 0); } /* * in handling the renew metermeta problem during insertion, + * * If the meter is created on demand during insertion, the routine usually waits for a short * period to re-issue the getMeterMeta msg, in which makes a greater change that vnode has * successfully created the corresponding table. @@ -2838,7 +3590,8 @@ static void tscWaitingForCreateTable(SSqlCmd *pCmd) { * @return status code */ int tscRenewMeterMeta(SSqlObj *pSql, char *meterId) { - int code = 0; + int code = 0; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0); // handle metric meta renew process SSqlCmd *pCmd = &pSql->cmd; @@ -2847,22 +3600,22 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *meterId) { if (pSql->fp == NULL) pSql->fp = (void *)0x1; /* - * 1. nly update the metermeta in force model metricmeta is not updated + * 1. only update the metermeta in force model metricmeta is not updated * 2. if get metermeta failed, still get the metermeta */ - if (pCmd->pMeterMeta == NULL || !tscQueryOnMetric(pCmd)) { - if (pCmd->pMeterMeta) { - tscTrace("%p update meter meta, old: numOfTags:%d, numOfCols:%d, uid:%d, addr:%p", - pSql, pCmd->pMeterMeta->numOfTags, pCmd->numOfCols, pCmd->pMeterMeta->uid, pCmd->pMeterMeta); + if (pMeterMetaInfo->pMeterMeta == NULL || !tscQueryOnMetric(pCmd)) { + if (pMeterMetaInfo->pMeterMeta) { + tscTrace("%p update meter meta, old: numOfTags:%d, numOfCols:%d, uid:%d, addr:%p", pSql, + pMeterMetaInfo->numOfTags, pCmd->numOfCols, pMeterMetaInfo->pMeterMeta->uid, pMeterMetaInfo->pMeterMeta); } - tscWaitingForCreateTable(&pSql->cmd); - taosRemoveDataFromCache(tscCacheHandle, (void **)&(pSql->cmd.pMeterMeta), true); + taosRemoveDataFromCache(tscCacheHandle, (void **)&(pMeterMetaInfo->pMeterMeta), true); - code = tscDoGetMeterMeta(pSql, meterId); + code = tscDoGetMeterMeta(pSql, meterId, 0); // todo ?? } else { - tscTrace("%p metric query not update metric meta, numOfTags:%d, numOfCols:%d, uid:%d, addr:%p", - pSql, pCmd->pMeterMeta->numOfTags, pCmd->numOfCols, pCmd->pMeterMeta->uid, pCmd->pMeterMeta); + tscTrace("%p metric query not update metric meta, numOfTags:%d, numOfCols:%d, uid:%d, addr:%p", pSql, + pMeterMetaInfo->pMeterMeta->numOfTags, pCmd->numOfCols, pMeterMetaInfo->pMeterMeta->uid, + pMeterMetaInfo->pMeterMeta); } if (code != TSDB_CODE_ACTION_IN_PROGRESS) { @@ -2874,44 +3627,65 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *meterId) { return code; } -int tscGetMetricMeta(SSqlObj *pSql, char *meterId) { - int code = TSDB_CODE_NETWORK_UNAVAIL; - char tagstr[TSDB_MAX_TAGS_LEN + 1] = {0}; +int tscGetMetricMeta(SSqlObj *pSql) { + int code = TSDB_CODE_NETWORK_UNAVAIL; + SSqlCmd *pCmd = &pSql->cmd; /* * the vnode query condition is serialized into pCmd->payload, we need to rebuild key for metricmeta info in cache. */ - tscGetMetricMetaCacheKey(&pSql->cmd, tagstr); - taosRemoveDataFromCache(tscCacheHandle, (void **)&(pSql->cmd.pMetricMeta), false); + bool reqMetricMeta = false; + for (int32_t i = 0; i < pSql->cmd.numOfTables; ++i) { + char tagstr[TSDB_MAX_TAGS_LEN + 1] = {0}; + + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, i); + tscGetMetricMetaCacheKey(pCmd, tagstr, pMeterMetaInfo->pMeterMeta->uid); + + taosRemoveDataFromCache(tscCacheHandle, (void **)&(pMeterMetaInfo->pMetricMeta), false); + + SMetricMeta *ppMeta = (SMetricMeta *)taosGetDataFromCache(tscCacheHandle, tagstr); + if (ppMeta == NULL) { + reqMetricMeta = true; + break; + } else { + pMeterMetaInfo->pMetricMeta = ppMeta; + } + } - SMetricMeta *ppMeta = (SMetricMeta *)taosGetDataFromCache(tscCacheHandle, tagstr); - if (ppMeta != NULL) { - pSql->cmd.pMetricMeta = ppMeta; + // all metricmeta are retrieved from cache, no need to query mgmt node + if (!reqMetricMeta) { return TSDB_CODE_SUCCESS; } - SSqlObj *pNew = malloc(sizeof(SSqlObj)); - memset(pNew, 0, sizeof(SSqlObj)); + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; pNew->cmd.command = TSDB_SQL_METRIC; - strcpy(pNew->cmd.name, meterId); - tscAllocPayloadWithSize(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE); + + for (int32_t i = 0; i < pSql->cmd.numOfTables; ++i) { + SMeterMetaInfo *pMMInfo = tscGetMeterMetaInfo(&pSql->cmd, i); + + SMeterMeta *pMeterMeta = taosGetDataFromCache(tscCacheHandle, pMMInfo->name); + tscAddMeterMetaInfo(&pNew->cmd, pMMInfo->name, pMeterMeta, NULL, pMMInfo->numOfTags, pMMInfo->tagColumnIndex); + } + + if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { + tscFreeSqlObj(pNew); + return code; + } // the query condition on meter is serialized into payload - tscTagCondAssign(&pNew->cmd.tagCond, &pSql->cmd.tagCond); + tscTagCondCopy(&pNew->cmd.tagCond, &pSql->cmd.tagCond); pNew->cmd.groupbyExpr = pSql->cmd.groupbyExpr; + pNew->cmd.numOfTables = pSql->cmd.numOfTables; - pNew->cmd.glimit = pSql->cmd.glimit; + pNew->cmd.slimit = pSql->cmd.slimit; pNew->cmd.order = pSql->cmd.order; - pNew->cmd.numOfReqTags = pSql->cmd.numOfReqTags; - - memcpy(pNew->cmd.tagColumnIndex, pSql->cmd.tagColumnIndex, sizeof(pSql->cmd.tagColumnIndex)); if (pSql->fp != NULL && pSql->pStream == NULL) { - tscfreeSqlCmdData(&pSql->cmd); + tscFreeSqlCmdData(&pSql->cmd); } tscTrace("%p allocate new pSqlObj:%p to get metricMeta", pSql, pNew); @@ -2920,7 +3694,21 @@ int tscGetMetricMeta(SSqlObj *pSql, char *meterId) { tsem_init(&pNew->emptyRspSem, 0, 1); code = tscProcessSql(pNew); - pSql->cmd.pMetricMeta = taosGetDataFromCache(tscCacheHandle, tagstr); + + for (int32_t i = 0; i < pCmd->numOfTables; ++i) { + char tagstr[TSDB_MAX_TAGS_LEN] = {0}; + + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, i); + tscGetMetricMetaCacheKey(pCmd, tagstr, pMeterMetaInfo->pMeterMeta->uid); + +#ifdef _DEBUG_VIEW + printf("create metric key:%s, index:%d\n", tagstr, i); +#endif + + taosRemoveDataFromCache(tscCacheHandle, (void **)&(pMeterMetaInfo->pMetricMeta), false); + pMeterMetaInfo->pMetricMeta = (SMetricMeta *)taosGetDataFromCache(tscCacheHandle, tagstr); + } + tscFreeSqlObj(pNew); } else { pNew->fp = tscMeterMetaCallBack; @@ -2951,9 +3739,9 @@ void tscInitMsgs() { tscBuildMsg[TSDB_SQL_DROP_DB] = tscBuildDropDbMsg; tscBuildMsg[TSDB_SQL_DROP_TABLE] = tscBuildDropTableMsg; tscBuildMsg[TSDB_SQL_ALTER_USER] = tscBuildAlterUserMsg; - tscBuildMsg[TSDB_SQL_CREATE_PNODE] = tscBuildCreateDnodeMsg; - tscBuildMsg[TSDB_SQL_DROP_PNODE] = tscBuildDropDnodeMsg; - tscBuildMsg[TSDB_SQL_CFG_PNODE] = tscBuildCfgDnodeMsg; + tscBuildMsg[TSDB_SQL_CREATE_DNODE] = tscBuildCreateDnodeMsg; + tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; + tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; @@ -2961,6 +3749,7 @@ void tscInitMsgs() { tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg; tscBuildMsg[TSDB_SQL_META] = tscBuildMeterMetaMsg; tscBuildMsg[TSDB_SQL_METRIC] = tscBuildMetricMetaMsg; + tscBuildMsg[TSDB_SQL_MULTI_META] = tscBuildMultiMeterMetaMsg; tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; @@ -2978,6 +3767,7 @@ void tscInitMsgs() { tscProcessMsgRsp[TSDB_SQL_USE_DB] = tscProcessUseDbRsp; tscProcessMsgRsp[TSDB_SQL_META] = tscProcessMeterMetaRsp; tscProcessMsgRsp[TSDB_SQL_METRIC] = tscProcessMetricMetaRsp; + tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiMeterMetaRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromMgmt; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 41eabec703..96489225d0 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -20,16 +20,18 @@ #include "tcache.h" #include "tlog.h" #include "trpc.h" +#include "tscJoinProcess.h" #include "tscProfile.h" #include "tscSecondaryMerge.h" #include "tscUtil.h" #include "tsclient.h" +#include "tscompression.h" #include "tsocket.h" #include "tsql.h" #include "ttimer.h" #include "tutil.h" -TAOS *taos_connect_imp(char *ip, char *user, char *pass, char *db, int port, void (*fp)(void *, TAOS_RES *, int), +TAOS *taos_connect_imp(const char *ip, const char *user, const char *pass, const char *db, int port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { STscObj *pObj; @@ -62,14 +64,30 @@ TAOS *taos_connect_imp(char *ip, char *user, char *pass, char *db, int port, voi } } +#ifdef CLUSTER + if (ip && ip[0]) { + tscMgmtIpList.numOfIps = 2; + strcpy(tscMgmtIpList.ipstr[0], ip); + tscMgmtIpList.ip[0] = inet_addr(ip); + + strcpy(tscMgmtIpList.ipstr[1], ip); + tscMgmtIpList.ip[1] = inet_addr(ip); + } +#else if (ip && ip[0]) { if (ip != tsServerIpStr) { strcpy(tsServerIpStr, ip); } tsServerIp = inet_addr(ip); } +#endif pObj = (STscObj *)malloc(sizeof(STscObj)); + if (NULL == pObj) { + globalCode = TSDB_CODE_CLI_OUT_OF_MEMORY; + return NULL; + } + memset(pObj, 0, sizeof(STscObj)); pObj->signature = pObj; @@ -96,6 +114,12 @@ TAOS *taos_connect_imp(char *ip, char *user, char *pass, char *db, int port, voi pthread_mutex_init(&pObj->mutex, NULL); SSqlObj *pSql = (SSqlObj *)malloc(sizeof(SSqlObj)); + if (NULL == pSql) { + globalCode = TSDB_CODE_CLI_OUT_OF_MEMORY; + free(pObj); + return NULL; + } + memset(pSql, 0, sizeof(SSqlObj)); pSql->pTscObj = pObj; pSql->signature = pSql; @@ -109,7 +133,13 @@ TAOS *taos_connect_imp(char *ip, char *user, char *pass, char *db, int port, voi } pSql->cmd.command = TSDB_SQL_CONNECT; - tscAllocPayloadWithSize(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE); + int ret = tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE); + if (TSDB_CODE_SUCCESS != ret) { + globalCode = TSDB_CODE_CLI_OUT_OF_MEMORY; + free(pSql); + free(pObj); + return NULL; + } pSql->res.code = tscProcessSql(pSql); if (fp != NULL) { @@ -126,28 +156,37 @@ TAOS *taos_connect_imp(char *ip, char *user, char *pass, char *db, int port, voi return pObj; } -TAOS *taos_connect(char *ip, char *user, char *pass, char *db, int port) { +TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, int port) { if (ip != NULL && (strcmp("127.0.0.1", ip) == 0 || strcasecmp("localhost", ip) == 0)) { +#ifdef CLUSTER + ip = tsPrivateIp; +#else ip = tsServerIpStr; +#endif } - - if (ip == NULL) ip = tsServerIpStr; tscTrace("try to create a connection to %s", ip); void *taos = taos_connect_imp(ip, user, pass, db, port, NULL, NULL, NULL); if (taos != NULL) { STscObj* pObj = (STscObj*) taos; + // version compare only requires the first 3 segments of the version string + int32_t comparedSegments = 3; + char client_version[64] = {0}; + char server_version[64] = {0}; int clientVersionNumber[4] = {0}; - if (!taosGetVersionNumber(version, clientVersionNumber)) { - tscError("taos:%p, invalid client version:%s", taos, version); + int serverVersionNumber[4] = {0}; + + strcpy(client_version, version); + strcpy(server_version, taos_get_server_info(taos)); + + if (!taosGetVersionNumber(client_version, clientVersionNumber)) { + tscError("taos:%p, invalid client version:%s", taos, client_version); pObj->pSql->res.code = TSDB_CODE_INVALID_CLIENT_VERSION; taos_close(taos); return NULL; } - char *server_version = taos_get_server_info(taos); - int serverVersionNumber[4] = {0}; if (!taosGetVersionNumber(server_version, serverVersionNumber)) { tscError("taos:%p, invalid server version:%s", taos, server_version); pObj->pSql->res.code = TSDB_CODE_INVALID_CLIENT_VERSION; @@ -155,9 +194,6 @@ TAOS *taos_connect(char *ip, char *user, char *pass, char *db, int port) { return NULL; } - // version compare only requires the first 3 segments of the version string - int32_t comparedSegments = 3; - for(int32_t i = 0; i < comparedSegments; ++i) { if (clientVersionNumber[i] != serverVersionNumber[i]) { tscError("taos:%p, the %d-th number of server version:%s not matched with client version:%s, close connection", @@ -174,17 +210,17 @@ TAOS *taos_connect(char *ip, char *user, char *pass, char *db, int port) { TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, int port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { +#ifndef CLUSTER if (ip == NULL) { ip = tsServerIpStr; } +#endif return taos_connect_imp(ip, user, pass, db, port, fp, param, taos); } void taos_close(TAOS *taos) { STscObj *pObj = (STscObj *)taos; - tscTrace("%p start to close connection, pSql:%p, HB:%p", pObj, pObj->pSql, pObj->pHb); - if (pObj == NULL) return; if (pObj->signature != pObj) return; @@ -195,37 +231,12 @@ void taos_close(TAOS *taos) { } } -int taos_query(TAOS *taos, char *sqlstr) { - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - globalCode = TSDB_CODE_DISCONNECTED; - return TSDB_CODE_DISCONNECTED; - } - - SSqlObj *pSql = pObj->pSql; +int taos_query_imp(STscObj* pObj, SSqlObj* pSql) { SSqlRes *pRes = &pSql->res; pRes->numOfRows = 1; pRes->numOfTotal = 0; - - tscTrace("%p SQL: %s pObj:%p", pSql, sqlstr, pObj); - - int32_t sqlLen = strlen(sqlstr); - if (sqlLen > TSDB_MAX_SQL_LEN) { - tscError("%p sql too long", pSql); - pRes->code = TSDB_CODE_INVALID_SQL; - return pRes->code; - } - - pSql->sqlstr = realloc(pSql->sqlstr, sqlLen + 1); - if (pSql->sqlstr == NULL) { - pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; - tscError("%p failed to malloc sql string buffer", pSql); - tscTrace("%p SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj); - return pRes->code; - } - - strtolower(pSql->sqlstr, sqlstr); + tscTrace("%p SQL: %s pObj:%p", pSql, pSql->sqlstr, pObj); pRes->code = (uint8_t)tsParseSql(pSql, pObj->acctId, pObj->db, false); @@ -241,7 +252,7 @@ int taos_query(TAOS *taos, char *sqlstr) { tscDoQuery(pSql); - tscTrace("%p SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj); + tscTrace("%p SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(pObj), pObj); if (pRes->code != TSDB_CODE_SUCCESS) { tscFreeSqlObjPartial(pSql); } @@ -249,6 +260,36 @@ int taos_query(TAOS *taos, char *sqlstr) { return pRes->code; } +int taos_query(TAOS *taos, const char *sqlstr) { + STscObj *pObj = (STscObj *)taos; + SSqlObj *pSql = pObj->pSql; + SSqlRes *pRes = &pSql->res; + + if (pObj == NULL || pObj->signature != pObj) { + globalCode = TSDB_CODE_DISCONNECTED; + return TSDB_CODE_DISCONNECTED; + } + + int32_t sqlLen = strlen(sqlstr); + if (sqlLen > TSDB_MAX_SQL_LEN) { + tscError("%p sql too long", pSql); + pRes->code = TSDB_CODE_INVALID_SQL; + return pRes->code; + } + + void *sql = realloc(pSql->sqlstr, sqlLen + 1); + if (sql == NULL) { + pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; + tscError("%p failed to malloc sql string buffer", pSql); + tscTrace("%p SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj); + return pRes->code; + } + + pSql->sqlstr = sql; + strtolower(pSql->sqlstr, sqlstr); + return taos_query_imp(pObj, pSql); +} + TAOS_RES *taos_use_result(TAOS *taos) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { @@ -272,14 +313,16 @@ int taos_num_fields(TAOS_RES *res) { SSqlObj *pSql = (SSqlObj *)res; if (pSql == NULL || pSql->signature != pSql) return 0; - return pSql->cmd.fieldsInfo.numOfOutputCols; + SFieldInfo *pFieldsInfo = &pSql->cmd.fieldsInfo; + + return (pFieldsInfo->numOfOutputCols - pFieldsInfo->numOfHiddenCols); } int taos_field_count(TAOS *taos) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) return 0; - return pObj->pSql->cmd.fieldsInfo.numOfOutputCols; + return taos_num_fields(pObj->pSql); } int taos_affected_rows(TAOS *taos) { @@ -304,9 +347,8 @@ int taos_retrieve(TAOS_RES *res) { if (pSql == NULL || pSql->signature != pSql) return 0; if (pRes->qhandle == 0) return 0; - pRes->row = 0; - pRes->numOfRows = 0; - pCmd->type = 0; + tscResetForNextRetrieve(pRes); + if (pCmd->command < TSDB_SQL_LOCAL) { pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; } @@ -327,9 +369,7 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) { } // Retrieve new block - pRes->row = 0; - pRes->numOfRows = 0; - pCmd->type = 0; + tscResetForNextRetrieve(pRes); if (pCmd->command < TSDB_SQL_LOCAL) { pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; } @@ -355,38 +395,15 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) { return (pCmd->order.order == TSQL_SO_DESC) ? pRes->numOfRows : -pRes->numOfRows; } -TAOS_ROW taos_fetch_row_impl(TAOS_RES *res) { - SSqlObj *pSql = (SSqlObj *)res; +static void **doSetResultRowData(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - STscObj *pObj = pSql->pTscObj; - int wccount = 0; - - if (pRes->qhandle == 0) return NULL; - if (pRes->row >= pRes->numOfRows) { - if (pObj->pSql != pSql) return NULL; - - pRes->row = 0; - pRes->numOfRows = 0; - pCmd->type = 0; - if (pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; - } - - tscProcessSql(pSql); - if (pRes->numOfRows == 0) { - return NULL; - } - - // secondary merge has handle this situation - if (pCmd->command != TSDB_SQL_RETRIEVE_METRIC) { - pRes->numOfTotal += pRes->numOfRows; - } - } + int32_t num = 0; for (int i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pCmd, i, pCmd->order) + pRes->bytes[i] * pRes->row; + // primary key column cannot be null in interval query, no need to check if (i == 0 && pCmd->nAggTimeInterval > 0) { continue; @@ -397,34 +414,169 @@ TAOS_ROW taos_fetch_row_impl(TAOS_RES *res) { if (isNull(pRes->tsrow[i], pField->type)) { pRes->tsrow[i] = NULL; } else if (pField->type == TSDB_DATA_TYPE_NCHAR) { - /* - * convert unicode to native code in a temporary buffer extra one byte for terminated symbol - */ - if (pRes->buffer[wccount] == NULL) { - pRes->buffer[wccount] = (char *)calloc(1, pField->bytes + 1); + // convert unicode to native code in a temporary buffer extra one byte for terminated symbol + if (pRes->buffer[num] == NULL) { + pRes->buffer[num] = malloc(pField->bytes + 1); } else { - pRes->buffer[wccount] = realloc(pRes->buffer[wccount], pField->bytes + 1); + pRes->buffer[num] = realloc(pRes->buffer[num], pField->bytes + 1); } /* string terminated */ - memset(pRes->buffer[wccount], 0, pField->bytes); + memset(pRes->buffer[num], 0, pField->bytes + 1); - if (taosUcs4ToMbs(pRes->tsrow[i], pField->bytes, pRes->buffer[wccount])) { - pRes->tsrow[i] = pRes->buffer[wccount]; + if (taosUcs4ToMbs(pRes->tsrow[i], pField->bytes, pRes->buffer[num])) { + pRes->tsrow[i] = pRes->buffer[num]; } else { tscError("%p charset:%s to %s. val:%ls convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, pRes->tsrow); pRes->tsrow[i] = NULL; } - wccount++; + num++; } } - assert(wccount <= pRes->numOfnchar); + assert(num <= pCmd->fieldsInfo.numOfOutputCols); + + return pRes->tsrow; +} + +static void **getOneRowFromBuf(SSqlObj *pSql) { + doSetResultRowData(pSql); + + SSqlRes *pRes = &pSql->res; pRes->row++; return pRes->tsrow; } +static void **tscJoinResultsetFromBuf(SSqlObj *pSql) { + SSqlCmd *pCmd = &pSql->cmd; + SSqlRes *pRes = &pSql->res; + + while (1) { + bool hasData = true; + + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + SSqlRes *pRes1 = &pSql->pSubs[i]->res; + + // in case inner join, if any subquery exhausted, query completed + if (pRes1->numOfRows == 0) { + hasData = false; + break; + } + } + + if (!hasData) { // free all sub sqlobj + tscTrace("%p one subquery exhausted, free other %d subquery", pSql, pSql->numOfSubs - 1); + + SSubqueryState *pState = NULL; + + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + SSqlObj * pChildObj = pSql->pSubs[i]; + SJoinSubquerySupporter *pSupporter = (SJoinSubquerySupporter *)pChildObj->param; + pState = pSupporter->pState; + + tscDestroyJoinSupporter(pChildObj->param); + taos_free_result(pChildObj); + } + + free(pState); + return NULL; + } + + if (pRes->tsrow == NULL) { + pRes->tsrow = malloc(sizeof(void *) * pCmd->exprsInfo.numOfExprs); + } + + bool success = false; + if (pSql->numOfSubs >= 2) { + // do merge result + SSqlRes *pRes1 = &pSql->pSubs[0]->res; + SSqlRes *pRes2 = &pSql->pSubs[1]->res; + + while (pRes1->row < pRes1->numOfRows && pRes2->row < pRes2->numOfRows) { + doSetResultRowData(pSql->pSubs[0]); + doSetResultRowData(pSql->pSubs[1]); + + TSKEY key1 = *(TSKEY *)pRes1->tsrow[0]; + TSKEY key2 = *(TSKEY *)pRes2->tsrow[0]; + + if (key1 == key2) { + success = true; + pRes1->row++; + pRes2->row++; + break; + } else if (key1 < key2) { + pRes1->row++; + } else if (key1 > key2) { + pRes2->row++; + } + } + } else { + SSqlRes *pRes1 = &pSql->pSubs[0]->res; + doSetResultRowData(pSql->pSubs[0]); + + success = (pRes1->row++ < pRes1->numOfRows); + } + + if (success) { + for (int32_t i = 0; i < pCmd->exprsInfo.numOfExprs; ++i) { + int32_t tableIndex = pRes->pColumnIndex[i].tableIndex; + int32_t columnIndex = pRes->pColumnIndex[i].columnIndex; + + SSqlRes *pRes1 = &pSql->pSubs[tableIndex]->res; + pRes->tsrow[i] = pRes1->tsrow[columnIndex]; + } + + break; + } else { + tscFetchDatablockFromSubquery(pSql); + if (pRes->code != TSDB_CODE_SUCCESS) { + return NULL; + } + } + } + + return pRes->tsrow; +} + +TAOS_ROW taos_fetch_row_impl(TAOS_RES *res) { + SSqlObj *pSql = (SSqlObj *)res; + SSqlCmd *pCmd = &pSql->cmd; + SSqlRes *pRes = &pSql->res; + + if (pRes->qhandle == 0 || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { + return NULL; + } + + if (pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE) { + tscFetchDatablockFromSubquery(pSql); + if (pRes->code == TSDB_CODE_SUCCESS) { + return tscJoinResultsetFromBuf(pSql); + } else { + return NULL; + } + + } else if (pRes->row >= pRes->numOfRows) { + tscResetForNextRetrieve(pRes); + + if (pCmd->command < TSDB_SQL_LOCAL) { + pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + } + + tscProcessSql(pSql); + if (pRes->numOfRows == 0) { + return NULL; + } + + // local reducer has handle this situation + if (pCmd->command != TSDB_SQL_RETRIEVE_METRIC) { + pRes->numOfTotal += pRes->numOfRows; + } + } + + return getOneRowFromBuf(pSql); +} + TAOS_ROW taos_fetch_row(TAOS_RES *res) { SSqlObj *pSql = (SSqlObj *)res; SSqlCmd *pCmd = &pSql->cmd; @@ -437,8 +589,10 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { // projection query on metric, pipeline retrieve data from vnode list, instead of two-stage merge TAOS_ROW rows = taos_fetch_row_impl(res); - while (rows == NULL && tscProjectionQueryOnMetric(pSql)) { - /* reach the maximum number of output rows, abort */ + while (rows == NULL && tscProjectionQueryOnMetric(pCmd)) { + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + + // reach the maximum number of output rows, abort if (pCmd->globalLimit > 0 && pRes->numOfTotal >= pCmd->globalLimit) { return NULL; } @@ -452,7 +606,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { assert((pRes->offset >= 0 && pRes->numOfRows == 0) || (pRes->offset == 0 && pRes->numOfRows >= 0)); - if ((++pCmd->vnodeIdx) <= pCmd->pMetricMeta->numOfVnodes) { + if ((++pCmd->vnodeIdx) < pMeterMetaInfo->pMetricMeta->numOfVnodes) { pCmd->command = TSDB_SQL_SELECT; assert(pSql->fp == NULL); tscProcessSql(pSql); @@ -460,7 +614,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { } // check!!! - if (rows != NULL || pCmd->vnodeIdx >= pCmd->pMetricMeta->numOfVnodes) { + if (rows != NULL || pCmd->vnodeIdx >= pMeterMetaInfo->pMetricMeta->numOfVnodes) { break; } } @@ -481,20 +635,26 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { return 0; } - // projection query on metric, pipeline retrieve data from vnode list, instead - // of two-stage mergevnodeProcessMsgFromShell free qhandle + // projection query on metric, pipeline retrieve data from vnode list, + // instead of two-stage mergevnodeProcessMsgFromShell free qhandle nRows = taos_fetch_block_impl(res, rows); - while (*rows == NULL && tscProjectionQueryOnMetric(pSql)) { + while (*rows == NULL && tscProjectionQueryOnMetric(pCmd)) { /* reach the maximum number of output rows, abort */ if (pCmd->globalLimit > 0 && pRes->numOfTotal >= pCmd->globalLimit) { return 0; } + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + /* update the limit value according to current retrieval results */ pCmd->limit.limit = pSql->cmd.globalLimit - pRes->numOfTotal; pCmd->limit.offset = pRes->offset; - if ((++pSql->cmd.vnodeIdx) <= pSql->cmd.pMetricMeta->numOfVnodes) { +#ifdef CLUSTER + if ((++pSql->cmd.vnodeIdx) <= pMeterMetaInfo->pMetricMeta->numOfVnodes) { +#else + if ((++pSql->cmd.vnodeIdx) < pMeterMetaInfo->pMetricMeta->numOfVnodes) { +#endif pSql->cmd.command = TSDB_SQL_SELECT; assert(pSql->fp == NULL); tscProcessSql(pSql); @@ -502,7 +662,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { } // check!!! - if (*rows != NULL || pCmd->vnodeIdx >= pCmd->pMetricMeta->numOfVnodes) { + if (*rows != NULL || pCmd->vnodeIdx >= pMeterMetaInfo->pMetricMeta->numOfVnodes) { break; } } @@ -510,7 +670,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { return nRows; } -int taos_select_db(TAOS *taos, char *db) { +int taos_select_db(TAOS *taos, const char *db) { char sql[64]; STscObj *pObj = (STscObj *)taos; @@ -534,6 +694,7 @@ void taos_free_result(TAOS_RES *res) { tscTrace("%p start to free result", pSql); if (pSql->signature != pSql) return; + if (pRes == NULL || pRes->qhandle == 0) { /* Query rsp is not received from vnode, so the qhandle is NULL */ tscTrace("%p qhandle is null, abort free, fp:%p", pSql, pSql->fp); @@ -547,52 +708,61 @@ void taos_free_result(TAOS_RES *res) { return; } - pCmd->type = 1; // set freeFlag to 1 in retrieve message if there are - // un-retrieved results + // set freeFlag to 1 in retrieve message if there are un-retrieved results + pCmd->type = TSDB_QUERY_TYPE_FREE_RESOURCE; + + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); /* - * case 1. Partial data have been retrieved from vnodes, but not all data has been retrieved yet. We need to recycle - * the connection by noticing the vnode return 0 results. + * case 1. Partial data have been retrieved from vnodes, but not all data has been retrieved yet. + * We need to recycle the connection by noticing the vnode return 0 results. * case 2. When the query response is received from vnodes and the numOfRows is set to 0, the user calls - * taos_free_result before the taos_fetch_row is called in non-stream computing, we need to recycle the - * connection. - * case 3. If the query process is cancelled by user in stable query, tscProcessSql should not be called for each - * subquery. Because the failure of execution tsProcessSql may trigger the callback function - * be executed, and the retry efforts may result in double free the - * resources, e.g.,SRetrieveSupport + * taos_free_result before the taos_fetch_row is called in non-stream computing, + * we need to recycle the connection. + * case 3. If the query process is cancelled by user in stable query, tscProcessSql should not be called + * for each subquery. Because the failure of execution tsProcessSql may trigger the callback function + * be executed, and the retry efforts may result in double free the resources, e.g.,SRetrieveSupport */ if (pRes->code != TSDB_CODE_QUERY_CANCELLED && ((pRes->numOfRows > 0 && pCmd->command < TSDB_SQL_LOCAL) || (pRes->code == TSDB_CODE_SUCCESS && pRes->numOfRows == 0 && pCmd->command == TSDB_SQL_SELECT && - pSql->pStream == NULL && pCmd->pMeterMeta != NULL))) { + pSql->pStream == NULL && pMeterMetaInfo->pMeterMeta != NULL))) { pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; - tscProcessSql(pSql); - if (pSql->fp) { + void *fp = pSql->fp; + if (fp != NULL) { pSql->freed = 1; - } else { - pSql->thandle = NULL; + } + tscProcessSql(pSql); + + /* + * If release connection msg is sent to vnode, the corresponding SqlObj for async query can not be freed instantly, + * since its free operation is delegated to callback function, which is tscProcessMsgFromServer. + */ + if (fp == NULL) { /* - * remove allocated resources and release metermeta/metricmeta references in cache - * since current query is completed - */ + * fp may be released here, so we cannot use the pSql->fp + * + * In case of handle sync model query, the main SqlObj cannot be freed. + * So, we only free part attributes, including allocated resources and references on metermeta/metricmeta + * data in cache. + * + * Then this object will be reused and no free operation is required. + */ + pSql->thandle = NULL; tscFreeSqlObjPartial(pSql); tscTrace("%p sql result is freed by app", pSql); } } else { + // if no free resource msg is sent to vnode, we free this object immediately. + pSql->thandle = NULL; + if (pSql->fp) { assert(pRes->numOfRows == 0 || (pCmd->command > TSDB_SQL_LOCAL)); - pSql->thandle = NULL; tscFreeSqlObj(pSql); - tscTrace("%p Async SqlObj is freed by app", pSql); + tscTrace("%p Async sql result is freed by app", pSql); } else { - pSql->thandle = NULL; - - /* - * remove allocated resources and release metermeta/metricmeta references in cache - * since current query is completed - */ tscFreeSqlObjPartial(pSql); tscTrace("%p sql result is freed", pSql); } @@ -658,7 +828,7 @@ void taos_stop_query(TAOS_RES *res) { pSql->res.code = TSDB_CODE_QUERY_CANCELLED; - if (tscIsTwoStageMergeMetricQuery(pSql)) { + if (tscIsTwoStageMergeMetricQuery(&pSql->cmd)) { tscKillMetricQuery(pSql); return; } @@ -710,7 +880,7 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) break; case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR:{ + case TSDB_DATA_TYPE_NCHAR: { /* limit the max length of string to no greater than the maximum length, * in case of not null-terminated string */ size_t xlen = strlen(row[i]); @@ -737,7 +907,7 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) return len; } -int taos_validate_sql(TAOS *taos, char *sql) { +int taos_validate_sql(TAOS *taos, const char *sql) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { globalCode = TSDB_CODE_DISCONNECTED; @@ -777,3 +947,141 @@ int taos_validate_sql(TAOS *taos, char *sql) { return code; } + +static int tscParseTblNameList(SSqlObj *pSql, const char* tblNameList, int32_t tblListLen) { + // must before clean the sqlcmd object + tscRemoveAllMeterMetaInfo(&pSql->cmd, false); + tscCleanSqlCmd(&pSql->cmd); + + SSqlCmd *pCmd = &pSql->cmd; + + pCmd->command = TSDB_SQL_MULTI_META; + pCmd->count = 0; + + int code = TSDB_CODE_INVALID_METER_ID; + char *str = (char*) tblNameList; + + SMeterMetaInfo *pMeterMetaInfo = tscAddEmptyMeterMetaInfo(pCmd); + + if ((code = tscAllocPayload(pCmd, tblListLen+16)) != TSDB_CODE_SUCCESS) { + return code; + } + + char *nextStr; + char tblName[TSDB_METER_ID_LEN]; + int payloadLen = 0; + char *pMsg = pCmd->payload; + while (1) { + nextStr = strchr(str, ','); + if (nextStr == NULL) { + break; + } + + memcpy(tblName, str, nextStr - str); + int32_t len = nextStr - str; + tblName[len] = '\0'; + + str = nextStr + 1; + + strtrim(tblName); + len = (uint32_t)strlen(tblName); + + SSQLToken sToken = {.n = len, .type = TK_ID, .z = tblName}; + tSQLGetToken(tblName, &sToken.type); + + // Check if the table name available or not + if (tscValidateName(&sToken) != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_INVALID_METER_ID; + sprintf(pCmd->payload, "table name is invalid"); + return code; + } + + if ((code = setMeterID(pSql, &sToken, 0)) != TSDB_CODE_SUCCESS) { + return code; + } + + if (++pCmd->count > TSDB_MULTI_METERMETA_MAX_NUM) { + code = TSDB_CODE_INVALID_METER_ID; + sprintf(pCmd->payload, "tables over the max number"); + return code; + } + + if (payloadLen + strlen(pMeterMetaInfo->name) + 128 >= pCmd->allocSize) { + char *pNewMem = realloc(pCmd->payload, pCmd->allocSize + tblListLen); + if (pNewMem == NULL) { + code = TSDB_CODE_CLI_OUT_OF_MEMORY; + sprintf(pCmd->payload, "failed to allocate memory"); + return code; + } + + pCmd->payload = pNewMem; + pCmd->allocSize = pCmd->allocSize + tblListLen; + pMsg = pCmd->payload; + } + + payloadLen += sprintf(pMsg + payloadLen, "%s,", pMeterMetaInfo->name); + } + + *(pMsg + payloadLen) = '\0'; + pCmd->payloadLen = payloadLen + 1; + + return TSDB_CODE_SUCCESS; +} + +int taos_load_table_info(TAOS *taos, const char *tableNameList) { + const int32_t MAX_TABLE_NAME_LENGTH = 12*1024*1024; // 12MB list + + STscObj *pObj = (STscObj *)taos; + if (pObj == NULL || pObj->signature != pObj) { + globalCode = TSDB_CODE_DISCONNECTED; + return TSDB_CODE_DISCONNECTED; + } + + SSqlObj *pSql = pObj->pSql; + SSqlRes *pRes = &pSql->res; + + pRes->numOfTotal = 0; // the number of getting table meta from server + pRes->code = 0; + + assert(pSql->fp == NULL); + tscTrace("%p tableNameList: %s pObj:%p", pSql, tableNameList, pObj); + + int32_t tblListLen = strlen(tableNameList); + if (tblListLen > MAX_TABLE_NAME_LENGTH) { + tscError("%p tableNameList too long, length:%d, maximum allowed:%d", pSql, tblListLen, MAX_TABLE_NAME_LENGTH); + pRes->code = TSDB_CODE_INVALID_SQL; + return pRes->code; + } + + char* str = calloc(1, tblListLen + 1); + if (str == NULL) { + pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; + tscError("%p failed to malloc sql string buffer", pSql); + return pRes->code; + } + + strtolower(str, tableNameList); + pRes->code = (uint8_t) tscParseTblNameList(pSql, str, tblListLen); + + /* + * set the qhandle to 0 before return in order to erase the qhandle value assigned in the previous successful query. + * If qhandle is NOT set 0, the function of taos_free_result() will send message to server by calling tscProcessSql() + * to free connection, which may cause segment fault, when the parse phrase is not even successfully executed. + */ + pRes->qhandle = 0; + pSql->thandle = NULL; + free(str); + + if (pRes->code != TSDB_CODE_SUCCESS) { + return pRes->code; + } + + tscDoQuery(pSql); + + tscTrace("%p load multi metermeta result:%d %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj); + if (pRes->code != TSDB_CODE_SUCCESS) { + tscFreeSqlObjPartial(pSql); + } + + return pRes->code; +} diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 4858898511..ec8233c6f9 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -13,12 +13,14 @@ * along with this program. If not, see . */ +#include "os.h" #include "tlog.h" #include "tsql.h" #include "ttime.h" #include "ttimer.h" #include "tutil.h" +#include "taosmsg.h" #include "tscUtil.h" #include "tsclient.h" @@ -32,7 +34,7 @@ static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer); static bool isProjectStream(SSqlCmd *pCmd) { for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pCmd, i); - if (pExpr->sqlFuncId != TSDB_FUNC_PRJ) { + if (pExpr->functionId != TSDB_FUNC_PRJ) { return false; } } @@ -64,14 +66,15 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { pSql->fp = tscProcessStreamQueryCallback; pSql->param = pStream; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0); - int code = tscGetMeterMeta(pSql, pSql->cmd.name); + int code = tscGetMeterMeta(pSql, pMeterMetaInfo->name, 0); pSql->res.code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; - if (code == 0 && UTIL_METER_IS_METRIC(&pSql->cmd)) { - code = tscGetMetricMeta(pSql, pSql->cmd.name); + if (code == 0 && UTIL_METER_IS_METRIC(pMeterMetaInfo)) { + code = tscGetMetricMeta(pSql); pSql->res.code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; @@ -88,7 +91,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { return; } - tscTrace("%p stream:%p start stream query on:%s", pSql, pStream, pSql->cmd.name); + tscTrace("%p stream:%p start stream query on:%s", pSql, pStream, pMeterMetaInfo->name); tscProcessSql(pStream->pSql); tscIncStreamExecutionCount(pStream); @@ -102,6 +105,7 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) { pStream->numOfRes = 0; // reset the numOfRes. SSqlObj *pSql = pStream->pSql; + tscTrace("%p add into timer", pSql); if (isProjectStream(&pSql->cmd)) { /* @@ -135,7 +139,9 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf tscError("%p stream:%p, query data failed, code:%d, retry in %lldms", pStream->pSql, pStream, numOfRows, retryDelay); - tscClearSqlMetaInfoForce(&(pStream->pSql->cmd)); + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(&pStream->pSql->cmd, 0); + tscClearMeterMetaInfo(pMeterMetaInfo, true); + tscSetRetryTimer(pStream, pStream->pSql, retryDelay); return; } @@ -143,7 +149,7 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf taos_fetch_rows_a(tres, tscProcessStreamRetrieveResult, param); } -static void tscSetTimestampForRes(SSqlStream *pStream, SSqlObj *pSql, int32_t numOfRows) { +static void tscSetTimestampForRes(SSqlStream *pStream, SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; int64_t timestamp = *(int64_t *)pRes->data; @@ -157,31 +163,29 @@ static void tscSetTimestampForRes(SSqlStream *pStream, SSqlObj *pSql, int32_t nu } static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOfRows) { - SSqlStream *pStream = (SSqlStream *)param; - SSqlObj * pSql = (SSqlObj *)res; + SSqlStream * pStream = (SSqlStream *)param; + SSqlObj * pSql = (SSqlObj *)res; + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0); if (pSql == NULL || numOfRows < 0) { int64_t retryDelayTime = tscGetRetryDelayTime(pStream->slidingTime, pStream->precision); tscError("%p stream:%p, retrieve data failed, code:%d, retry in %lldms", pSql, pStream, numOfRows, retryDelayTime); - tscClearSqlMetaInfoForce(&(pStream->pSql->cmd)); + tscClearMeterMetaInfo(pMeterMetaInfo, true); tscSetRetryTimer(pStream, pStream->pSql, retryDelayTime); return; } - if (numOfRows > 0) { // save - // when reaching here the first execution of stream computing is successful. + if (numOfRows > 0) { // when reaching here the first execution of stream computing is successful. pStream->numOfRes += numOfRows; - TAOS_ROW row = NULL; //; - while ((row = taos_fetch_row(res)) != NULL) { - // char result[512] = {0}; - // taos_print_row(result, row, pSql->cmd.fieldsInfo.pFields, pSql->cmd.fieldsInfo.numOfOutputCols); - // tscPrint("%p stream:%p query result: %s", pSql, pStream, result); + + for(int32_t i = 0; i < numOfRows; ++i) { + TAOS_ROW row = taos_fetch_row(res); tscTrace("%p stream:%p fetch result", pSql, pStream); if (isProjectStream(&pSql->cmd)) { pStream->stime = *(TSKEY *)row[0]; } else { - tscSetTimestampForRes(pStream, pSql, numOfRows); + tscSetTimestampForRes(pStream, pSql); } // user callback function @@ -214,7 +218,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf row[i] = pSql->res.data + offset; } - tscSetTimestampForRes(pStream, pSql, numOfRows); + tscSetTimestampForRes(pStream, pSql); row[0] = pRes->data; // char result[512] = {0}; @@ -243,11 +247,11 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf } } - tscTrace("%p stream:%p, query on:%s, fetch result completed, fetched rows:%d.", pSql, pStream, pSql->cmd.name, + tscTrace("%p stream:%p, query on:%s, fetch result completed, fetched rows:%d", pSql, pStream, pMeterMetaInfo->name, pStream->numOfRes); - /* release the metric/meter meta information reference, so data in cache can be updated */ - tscClearSqlMetaInfo(&(pSql->cmd)); + // release the metric/meter meta information reference, so data in cache can be updated + tscClearMeterMetaInfo(pMeterMetaInfo, false); tscSetNextLaunchTimer(pStream, pSql); } } @@ -351,7 +355,6 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) { if (pCmd->nAggTimeInterval < minIntervalTime) { tscWarn("%p stream:%p, original sample interval:%ld too small, reset to:%lld", pSql, pStream, pCmd->nAggTimeInterval, minIntervalTime); - pCmd->nAggTimeInterval = minIntervalTime; } @@ -447,7 +450,7 @@ static void setErrorInfo(STscObj* pObj, int32_t code, char* info) { strncpy(pCmd->payload, info, pCmd->payloadLen); } -TAOS_STREAM *taos_open_stream(TAOS *taos, char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), +TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), int64_t stime, void *param, void (*callback)(void *)) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) return NULL; @@ -462,7 +465,12 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, char *sqlstr, void (*fp)(void *param, pSql->pTscObj = pObj; SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - tscAllocPayloadWithSize(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); + int ret = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); + if (TSDB_CODE_SUCCESS != ret) { + setErrorInfo(pObj, ret, NULL); + free(pSql); + return NULL; + } pSql->sqlstr = strdup(sqlstr); if (pSql->sqlstr == NULL) { @@ -479,10 +487,16 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, char *sqlstr, void (*fp)(void *param, tSQLParse(&SQLInfo, pSql->sqlstr); tscCleanSqlCmd(&pSql->cmd); - tscAllocPayloadWithSize(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE); - - //todo refactor later - pSql->cmd.count = 1; + ret = tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE); + if (TSDB_CODE_SUCCESS != ret) { + setErrorInfo(pObj, ret, NULL); + tscError("%p open stream failed, sql:%s, code:%d", pSql, sqlstr, TSDB_CODE_CLI_OUT_OF_MEMORY); + tscFreeSqlObj(pSql); + return NULL; + } + + // TODO later refactor use enum + pSql->cmd.count = 1; // 1 means sql in stream, allowed the sliding clause. pRes->code = tscToSQLCmd(pSql, &SQLInfo); SQLInfoDestroy(&SQLInfo); @@ -503,13 +517,16 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, char *sqlstr, void (*fp)(void *param, return NULL; } + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + pStream->fp = fp; pStream->callback = callback; pStream->param = param; pStream->pSql = pSql; - pStream->ctime = taosGetTimestamp(pCmd->pMeterMeta->precision); + pStream->precision = pMeterMetaInfo->pMeterMeta->precision; + + pStream->ctime = taosGetTimestamp(pStream->precision); pStream->etime = pCmd->etime; - pStream->precision = pCmd->pMeterMeta->precision; pSql->pStream = pStream; tscAddIntoStreamList(pStream); @@ -521,7 +538,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, char *sqlstr, void (*fp)(void *param, taosTmrReset(tscProcessStreamTimer, starttime, pStream, tscTmr, &pStream->pTimer); tscTrace("%p stream:%p is opened, query on:%s, interval:%lld, sliding:%lld, first launched in:%lld, sql:%s", pSql, - pStream, pSql->cmd.name, pStream->interval, pStream->slidingTime, starttime, sqlstr); + pStream, pMeterMetaInfo->name, pStream->interval, pStream->slidingTime, starttime, sqlstr); return pStream; } diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index c801eac740..dee8f02118 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -36,7 +36,7 @@ typedef struct { TAOS_RES * result; } SSub; -TAOS_SUB *taos_subscribe(char *host, char *user, char *pass, char *db, char *name, int64_t time, int mseconds) { +TAOS_SUB *taos_subscribe(const char *host, const char *user, const char *pass, const char *db, const char *name, int64_t time, int mseconds) { SSub *pSub; pSub = (SSub *)malloc(sizeof(SSub)); diff --git a/src/client/src/tscSyntaxtreefunction.c b/src/client/src/tscSyntaxtreefunction.c index f231d13935..00781919e4 100644 --- a/src/client/src/tscSyntaxtreefunction.c +++ b/src/client/src/tscSyntaxtreefunction.c @@ -23,39 +23,36 @@ #include "ttypes.h" #include "tutil.h" -#define ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, op, _res_type, _ord) \ - { \ - int32_t i = ((_ord) == TSQL_SO_ASC) ? 0 : MAX(len1, len2) - 1; \ - int32_t step = ((_ord) == TSQL_SO_ASC) ? 1 : -1; \ - \ - if ((len1) == (len2)) { \ - for (; i < (len2) && i >= 0; i += step) { \ - if (isNull((char *)&(left)[i], _left_type) || isNull((char *)&(right)[i], _right_type)) { \ - setNull((char *)&(out)[i], _res_type, tDataTypeDesc[_res_type].nSize); \ - continue; \ - } \ - *(out) = (double)(left)[i] op(right)[i]; \ - (out) += step; \ - } \ - } else if ((len1) == 1) { \ - for (; i >= 0 && i < (len2); i += step) { \ - if (isNull((char *)&(left)[i], _left_type) || isNull((char *)&(right)[i], _right_type)) { \ - setNull((char *)&(out)[i], _res_type, tDataTypeDesc[_res_type].nSize); \ - continue; \ - } \ - *(out) = (double)pLeft[0] op(pRight)[i]; \ - (out) += step; \ - } \ - } else if ((len2) == 1) { \ - for (; i >= 0 && i < (len1); i += step) { \ - if (isNull((char *)&(left)[i], _left_type) || isNull((char *)&(right)[i], _right_type)) { \ - setNull((char *)&(out)[i], _res_type, tDataTypeDesc[_res_type].nSize); \ - continue; \ - } \ - *(out) = (double)(pLeft)[i] op(pRight)[0]; \ - (out) += step; \ - } \ - } \ +#define ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, op, _res_type, _ord) \ + { \ + int32_t i = ((_ord) == TSQL_SO_ASC) ? 0 : MAX(len1, len2) - 1; \ + int32_t step = ((_ord) == TSQL_SO_ASC) ? 1 : -1; \ + \ + if ((len1) == (len2)) { \ + for (; i < (len2) && i >= 0; i += step, (out) += step) { \ + if (isNull((char *)&((left)[i]), _left_type) || isNull((char *)&((right)[i]), _right_type)) { \ + setNull((char *)(out), _res_type, tDataTypeDesc[_res_type].nSize); \ + continue; \ + } \ + *(out) = (double)(left)[i] op(right)[i]; \ + } \ + } else if ((len1) == 1) { \ + for (; i >= 0 && i < (len2); i += step, (out) += step) { \ + if (isNull((char *)(left), _left_type) || isNull((char *)&(right)[i], _right_type)) { \ + setNull((char *)(out), _res_type, tDataTypeDesc[_res_type].nSize); \ + continue; \ + } \ + *(out) = (double)(left)[0] op(right)[i]; \ + } \ + } else if ((len2) == 1) { \ + for (; i >= 0 && i < (len1); i += step, (out) += step) { \ + if (isNull((char *)&(left)[i], _left_type) || isNull((char *)(right), _right_type)) { \ + setNull((char *)(out), _res_type, tDataTypeDesc[_res_type].nSize); \ + continue; \ + } \ + *(out) = (double)(left)[i] op(right)[0]; \ + } \ + } \ } #define ARRAY_LIST_OP_REM(left, right, _left_type, _right_type, len1, len2, out, op, _res_type, _ord) \ @@ -64,31 +61,28 @@ int32_t step = (_ord == TSQL_SO_ASC) ? 1 : -1; \ \ if (len1 == (len2)) { \ - for (; i >= 0 && i < (len2); i += step) { \ - if (isNull((char *)&left[i], _left_type) || isNull((char *)&right[i], _right_type)) { \ - setNull((char *)&out[i], _res_type, tDataTypeDesc[_res_type].nSize); \ + for (; i >= 0 && i < (len2); i += step, (out) += step) { \ + if (isNull((char *)&(left[i]), _left_type) || isNull((char *)&(right[i]), _right_type)) { \ + setNull((char *)(out), _res_type, tDataTypeDesc[_res_type].nSize); \ continue; \ } \ - *(out) = (double)pLeft[i] - ((int64_t)(((double)pLeft[i]) / pRight[i])) * pRight[i]; \ - (out) += step; \ + *(out) = (double)(left)[i] - ((int64_t)(((double)(left)[i]) / (right)[i])) * (right)[i]; \ } \ } else if (len1 == 1) { \ - for (; i >= 0 && i < (len2); i += step) { \ - if (isNull((char *)&left[i], _left_type) || isNull((char *)&right[i], _right_type)) { \ - setNull((char *)&out[i], _res_type, tDataTypeDesc[_res_type].nSize); \ + for (; i >= 0 && i < (len2); i += step, (out) += step) { \ + if (isNull((char *)(left), _left_type) || isNull((char *)&((right)[i]), _right_type)) { \ + setNull((char *)(out), _res_type, tDataTypeDesc[_res_type].nSize); \ continue; \ } \ - *(out) = (double)pLeft[0] - ((int64_t)(((double)pLeft[0]) / pRight[i])) * pRight[i]; \ - (out) += step; \ + *(out) = (double)(left)[0] - ((int64_t)(((double)(left)[0]) / (right)[i])) * (right)[i]; \ } \ } else if ((len2) == 1) { \ - for (; i >= 0 && i < len1; i += step) { \ - if (isNull((char *)&left[i], _left_type) || isNull((char *)&right[i], _right_type)) { \ - setNull((char *)&out[i], _res_type, tDataTypeDesc[_res_type].nSize); \ + for (; i >= 0 && i < len1; i += step, (out) += step) { \ + if (isNull((char *)&((left)[i]), _left_type) || isNull((char *)(right), _right_type)) { \ + setNull((char *)(out), _res_type, tDataTypeDesc[_res_type].nSize); \ continue; \ } \ - *(out) = (double)pLeft[i] - ((int64_t)(((double)pLeft[i]) / pRight[0])) * pRight[0]; \ - (out) += step; \ + *(out) = (double)(left)[i] - ((int64_t)(((double)(left)[i]) / (right)[0])) * (right)[0]; \ } \ } \ } @@ -121,33 +115,30 @@ void calc_fn_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRi int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_INT)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numRight; i += step, pOutput += step) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } *pOutput = (double)pLeft[i] + pRight[i]; - pOutput += step; } } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; ++i) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_INT)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numRight; i += step, pOutput += step) { + if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } *pOutput = (double)pLeft[0] + pRight[i]; - pOutput += step; } } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; ++i) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_INT)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numLeft; i += step, pOutput += step) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { + setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } *pOutput = (double)pLeft[i] + pRight[0]; - pOutput += step; } } } @@ -322,31 +313,28 @@ void calc_fn_i32_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRi int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_INT)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numRight; i += step, pOutput += step) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + setNull((char *)&(pOutput[i]), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } *pOutput = (double)pLeft[i] - pRight[i]; - pOutput += step; } } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_INT)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numRight; i += step, pOutput += step) { + if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } *pOutput = (double)pLeft[0] - pRight[i]; - pOutput += step; } } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_INT)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numLeft; i += step, pOutput += step) { + if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { + setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } *pOutput = (double)pLeft[i] - pRight[0]; - pOutput += step; } } } @@ -536,31 +524,30 @@ void calc_fn_i32_i32_multi(void *left, void *right, int32_t numLeft, int32_t num int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_INT)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numRight; i += step, pOutput += step) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } + *pOutput = (double)pLeft[i] * pRight[i]; - pOutput += step; } } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_INT)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numRight; i += step, pOutput += step) { + if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } + *pOutput = (double)pLeft[0] * pRight[i]; - pOutput += step; } } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_INT)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numLeft; i += step, pOutput += step) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { + setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } *pOutput = (double)pLeft[i] * pRight[0]; - pOutput += step; } } } @@ -735,31 +722,30 @@ void calc_fn_i32_i32_div(void *left, void *right, int32_t numLeft, int32_t numRi int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_INT)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numRight; i += step, pOutput += step) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } + *pOutput = (double)pLeft[i] / pRight[i]; - pOutput += step; } } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_INT)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numRight; i += step, pOutput += step) { + if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } + *pOutput = (double)pLeft[0] / pRight[i]; - pOutput += step; } } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_INT)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numLeft; i += step, pOutput += step) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { + setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } *pOutput = (double)pLeft[i] / pRight[0]; - pOutput += step; } } } @@ -950,32 +936,31 @@ void calc_fn_i32_i32_rem(void *left, void *right, int32_t numLeft, int32_t numRi int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_INT)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numRight; i += step, pOutput += step) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } *pOutput = (double)pLeft[i] - ((int64_t)(((double)pLeft[i]) / pRight[i])) * pRight[i]; - pOutput += step; } } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_INT)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numRight; i += step, pOutput += step) { + if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } + *pOutput = (double)pLeft[0] - ((int64_t)(((double)pLeft[0]) / pRight[i])) * pRight[i]; - pOutput += step; } } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_INT)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numLeft; i += step, pOutput += step) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { + setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } + *pOutput = (double)pLeft[i] - ((int64_t)(((double)pLeft[i]) / pRight[0])) * pRight[0]; - pOutput += step; } } } @@ -1009,31 +994,31 @@ void calc_fn_i32_d_rem(void *left, void *right, int32_t numLeft, int32_t numRigh int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_DOUBLE)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numRight; i += step, pOutput += step) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } + *pOutput = (double)pLeft[i] - ((int64_t)(((double)pLeft[i]) / pRight[i])) * pRight[i]; - pOutput += step; } } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_DOUBLE)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numRight; i += step, pOutput += step) { + if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } + *pOutput = (double)pLeft[0] - ((int64_t)(((double)pLeft[0]) / pRight[i])) * pRight[i]; - pOutput += step; } } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)&pRight[i], TSDB_DATA_TYPE_DOUBLE)) { - setNull((char *)&pOutput[i], TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + for (; i >= 0 && i < numLeft; i += step, pOutput += step) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { + setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } + *pOutput = (double)pLeft[i] - ((int64_t)(((double)pLeft[i]) / pRight[0])) * pRight[0]; - pOutput += step; } } } @@ -1192,98 +1177,63 @@ void calc_fn_d_d_rem(void *left, void *right, int32_t numLeft, int32_t numRight, * the following are two-dimensional array list of callback function . */ _bi_consumer_fn_t add_function_arraylist[8][10] = { - /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, - binary*/ + /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, binary*/ {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // EMPTY, {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // TSDB_DATA_TYPE_BOOL, - {NULL, NULL, calc_fn_i8_i8_add, calc_fn_i8_i16_add, calc_fn_i8_i32_add, calc_fn_i8_i64_add, calc_fn_i8_f_add, - calc_fn_i8_d_add, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT - {NULL, NULL, calc_fn_i16_i8_add, calc_fn_i16_i16_add, calc_fn_i16_i32_add, calc_fn_i16_i64_add, calc_fn_i16_f_add, - calc_fn_i16_d_add, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT - {NULL, NULL, calc_fn_i32_i8_add, calc_fn_i32_i16_add, calc_fn_i32_i32_add, calc_fn_i32_i64_add, calc_fn_i32_f_add, - calc_fn_i32_d_add, NULL, NULL}, // TSDB_DATA_TYPE_INT - {NULL, NULL, calc_fn_i64_i8_add, calc_fn_i64_i16_add, calc_fn_i64_i32_add, calc_fn_i64_i64_add, calc_fn_i64_f_add, - calc_fn_i64_d_add, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT - {NULL, NULL, calc_fn_f_i8_add, calc_fn_f_i16_add, calc_fn_f_i32_add, calc_fn_f_i64_add, calc_fn_f_f_add, - calc_fn_f_d_add, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT - {NULL, NULL, calc_fn_d_i8_add, calc_fn_d_i16_add, calc_fn_d_i32_add, calc_fn_d_i64_add, calc_fn_d_f_add, - calc_fn_d_d_add, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE + {NULL, NULL, calc_fn_i8_i8_add, calc_fn_i8_i16_add, calc_fn_i8_i32_add, calc_fn_i8_i64_add, calc_fn_i8_f_add, calc_fn_i8_d_add, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT + {NULL, NULL, calc_fn_i16_i8_add, calc_fn_i16_i16_add, calc_fn_i16_i32_add, calc_fn_i16_i64_add, calc_fn_i16_f_add, calc_fn_i16_d_add, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT + {NULL, NULL, calc_fn_i32_i8_add, calc_fn_i32_i16_add, calc_fn_i32_i32_add, calc_fn_i32_i64_add, calc_fn_i32_f_add, calc_fn_i32_d_add, NULL, NULL}, // TSDB_DATA_TYPE_INT + {NULL, NULL, calc_fn_i64_i8_add, calc_fn_i64_i16_add, calc_fn_i64_i32_add, calc_fn_i64_i64_add, calc_fn_i64_f_add, calc_fn_i64_d_add, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT + {NULL, NULL, calc_fn_f_i8_add, calc_fn_f_i16_add, calc_fn_f_i32_add, calc_fn_f_i64_add, calc_fn_f_f_add, calc_fn_f_d_add, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT + {NULL, NULL, calc_fn_d_i8_add, calc_fn_d_i16_add, calc_fn_d_i32_add, calc_fn_d_i64_add, calc_fn_d_f_add, calc_fn_d_d_add, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE }; _bi_consumer_fn_t sub_function_arraylist[8][10] = { - /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, - binary*/ + /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, binary*/ {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // EMPTY, {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // TSDB_DATA_TYPE_BOOL, - {NULL, NULL, calc_fn_i8_i8_sub, calc_fn_i8_i16_sub, calc_fn_i8_i32_sub, calc_fn_i8_i64_sub, calc_fn_i8_f_sub, - calc_fn_i8_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT - {NULL, NULL, calc_fn_i16_i8_sub, calc_fn_i16_i16_sub, calc_fn_i16_i32_sub, calc_fn_i16_i64_sub, calc_fn_i16_f_sub, - calc_fn_i16_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT - {NULL, NULL, calc_fn_i32_i8_sub, calc_fn_i32_i16_sub, calc_fn_i32_i32_sub, calc_fn_i32_i64_sub, calc_fn_i32_f_sub, - calc_fn_i32_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_INT - {NULL, NULL, calc_fn_i64_i8_sub, calc_fn_i64_i16_sub, calc_fn_i64_i32_sub, calc_fn_i64_i64_sub, calc_fn_i64_f_sub, - calc_fn_i64_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT - {NULL, NULL, calc_fn_f_i8_sub, calc_fn_f_i16_sub, calc_fn_f_i32_sub, calc_fn_f_i64_sub, calc_fn_f_f_sub, - calc_fn_f_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT - {NULL, NULL, calc_fn_d_i8_sub, calc_fn_d_i16_sub, calc_fn_d_i32_sub, calc_fn_d_i64_sub, calc_fn_d_f_sub, - calc_fn_d_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE + {NULL, NULL, calc_fn_i8_i8_sub, calc_fn_i8_i16_sub, calc_fn_i8_i32_sub, calc_fn_i8_i64_sub, calc_fn_i8_f_sub, calc_fn_i8_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT + {NULL, NULL, calc_fn_i16_i8_sub, calc_fn_i16_i16_sub, calc_fn_i16_i32_sub, calc_fn_i16_i64_sub, calc_fn_i16_f_sub, calc_fn_i16_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT + {NULL, NULL, calc_fn_i32_i8_sub, calc_fn_i32_i16_sub, calc_fn_i32_i32_sub, calc_fn_i32_i64_sub, calc_fn_i32_f_sub, calc_fn_i32_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_INT + {NULL, NULL, calc_fn_i64_i8_sub, calc_fn_i64_i16_sub, calc_fn_i64_i32_sub, calc_fn_i64_i64_sub, calc_fn_i64_f_sub, calc_fn_i64_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT + {NULL, NULL, calc_fn_f_i8_sub, calc_fn_f_i16_sub, calc_fn_f_i32_sub, calc_fn_f_i64_sub, calc_fn_f_f_sub, calc_fn_f_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT + {NULL, NULL, calc_fn_d_i8_sub, calc_fn_d_i16_sub, calc_fn_d_i32_sub, calc_fn_d_i64_sub, calc_fn_d_f_sub, calc_fn_d_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE }; _bi_consumer_fn_t multi_function_arraylist[][10] = { - /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, - binary*/ + /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, binary*/ {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // EMPTY, {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // TSDB_DATA_TYPE_BOOL, - {NULL, NULL, calc_fn_i8_i8_multi, calc_fn_i8_i16_multi, calc_fn_i8_i32_multi, calc_fn_i8_i64_multi, - calc_fn_i8_f_multi, calc_fn_i8_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT - {NULL, NULL, calc_fn_i16_i8_multi, calc_fn_i16_i16_multi, calc_fn_i16_i32_multi, calc_fn_i16_i64_multi, - calc_fn_i16_f_multi, calc_fn_i16_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT - {NULL, NULL, calc_fn_i32_i8_multi, calc_fn_i32_i16_multi, calc_fn_i32_i32_multi, calc_fn_i32_i64_multi, - calc_fn_i32_f_multi, calc_fn_i32_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_INT - {NULL, NULL, calc_fn_i64_i8_multi, calc_fn_i64_i16_multi, calc_fn_i64_i32_multi, calc_fn_i64_i64_multi, - calc_fn_i64_f_multi, calc_fn_i64_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT - {NULL, NULL, calc_fn_f_i8_multi, calc_fn_f_i16_multi, calc_fn_f_i32_multi, calc_fn_f_i64_multi, calc_fn_f_f_multi, - calc_fn_f_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT - {NULL, NULL, calc_fn_d_i8_multi, calc_fn_d_i16_multi, calc_fn_d_i32_multi, calc_fn_d_i64_multi, calc_fn_d_f_multi, - calc_fn_d_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE + {NULL, NULL, calc_fn_i8_i8_multi, calc_fn_i8_i16_multi, calc_fn_i8_i32_multi, calc_fn_i8_i64_multi, calc_fn_i8_f_multi, calc_fn_i8_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT + {NULL, NULL, calc_fn_i16_i8_multi, calc_fn_i16_i16_multi, calc_fn_i16_i32_multi, calc_fn_i16_i64_multi, calc_fn_i16_f_multi, calc_fn_i16_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT + {NULL, NULL, calc_fn_i32_i8_multi, calc_fn_i32_i16_multi, calc_fn_i32_i32_multi, calc_fn_i32_i64_multi, calc_fn_i32_f_multi, calc_fn_i32_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_INT + {NULL, NULL, calc_fn_i64_i8_multi, calc_fn_i64_i16_multi, calc_fn_i64_i32_multi, calc_fn_i64_i64_multi, calc_fn_i64_f_multi, calc_fn_i64_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT + {NULL, NULL, calc_fn_f_i8_multi, calc_fn_f_i16_multi, calc_fn_f_i32_multi, calc_fn_f_i64_multi, calc_fn_f_f_multi, calc_fn_f_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT + {NULL, NULL, calc_fn_d_i8_multi, calc_fn_d_i16_multi, calc_fn_d_i32_multi, calc_fn_d_i64_multi, calc_fn_d_f_multi, calc_fn_d_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE }; _bi_consumer_fn_t div_function_arraylist[8][10] = { - /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, - binary*/ + /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, binary*/ {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // EMPTY, {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // TSDB_DATA_TYPE_BOOL, - {NULL, NULL, calc_fn_i8_i8_div, calc_fn_i8_i16_div, calc_fn_i8_i32_div, calc_fn_i8_i64_div, calc_fn_i8_f_div, - calc_fn_i8_d_div, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT - {NULL, NULL, calc_fn_i16_i8_div, calc_fn_i16_i16_div, calc_fn_i16_i32_div, calc_fn_i16_i64_div, calc_fn_i16_f_div, - calc_fn_i16_d_div, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT - {NULL, NULL, calc_fn_i32_i8_div, calc_fn_i32_i16_div, calc_fn_i32_i32_div, calc_fn_i32_i64_div, calc_fn_i32_f_div, - calc_fn_i32_d_div, NULL, NULL}, // TSDB_DATA_TYPE_INT - {NULL, NULL, calc_fn_i64_i8_div, calc_fn_i64_i16_div, calc_fn_i64_i32_div, calc_fn_i64_i64_div, calc_fn_i64_f_div, - calc_fn_i64_d_div, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT - {NULL, NULL, calc_fn_f_i8_div, calc_fn_f_i16_div, calc_fn_f_i32_div, calc_fn_f_i64_div, calc_fn_f_f_div, - calc_fn_f_d_div, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT - {NULL, NULL, calc_fn_d_i8_div, calc_fn_d_i16_div, calc_fn_d_i32_div, calc_fn_d_i64_div, calc_fn_d_f_div, - calc_fn_d_d_div, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE + {NULL, NULL, calc_fn_i8_i8_div, calc_fn_i8_i16_div, calc_fn_i8_i32_div, calc_fn_i8_i64_div, calc_fn_i8_f_div, calc_fn_i8_d_div, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT + {NULL, NULL, calc_fn_i16_i8_div, calc_fn_i16_i16_div, calc_fn_i16_i32_div, calc_fn_i16_i64_div, calc_fn_i16_f_div, calc_fn_i16_d_div, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT + {NULL, NULL, calc_fn_i32_i8_div, calc_fn_i32_i16_div, calc_fn_i32_i32_div, calc_fn_i32_i64_div, calc_fn_i32_f_div, calc_fn_i32_d_div, NULL, NULL}, // TSDB_DATA_TYPE_INT + {NULL, NULL, calc_fn_i64_i8_div, calc_fn_i64_i16_div, calc_fn_i64_i32_div, calc_fn_i64_i64_div, calc_fn_i64_f_div, calc_fn_i64_d_div, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT + {NULL, NULL, calc_fn_f_i8_div, calc_fn_f_i16_div, calc_fn_f_i32_div, calc_fn_f_i64_div, calc_fn_f_f_div, calc_fn_f_d_div, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT + {NULL, NULL, calc_fn_d_i8_div, calc_fn_d_i16_div, calc_fn_d_i32_div, calc_fn_d_i64_div, calc_fn_d_f_div, calc_fn_d_d_div, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE }; _bi_consumer_fn_t rem_function_arraylist[8][10] = { - /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, - binary*/ + /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, binary*/ {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // EMPTY, {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // TSDB_DATA_TYPE_BOOL, - {NULL, NULL, calc_fn_i8_i8_rem, calc_fn_i8_i16_rem, calc_fn_i8_i32_rem, calc_fn_i8_i64_rem, calc_fn_i8_f_rem, - calc_fn_i8_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT - {NULL, NULL, calc_fn_i16_i8_rem, calc_fn_i16_i16_rem, calc_fn_i16_i32_rem, calc_fn_i16_i64_rem, calc_fn_i16_f_rem, - calc_fn_i16_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT - {NULL, NULL, calc_fn_i32_i8_rem, calc_fn_i32_i16_rem, calc_fn_i32_i32_rem, calc_fn_i32_i64_rem, calc_fn_i32_f_rem, - calc_fn_i32_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_INT - {NULL, NULL, calc_fn_i64_i8_rem, calc_fn_i64_i16_rem, calc_fn_i64_i32_rem, calc_fn_i64_i64_rem, calc_fn_i64_f_rem, - calc_fn_i64_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT - {NULL, NULL, calc_fn_f_i8_rem, calc_fn_f_i16_rem, calc_fn_f_i32_rem, calc_fn_f_i64_rem, calc_fn_f_f_rem, - calc_fn_f_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT - {NULL, NULL, calc_fn_d_i8_rem, calc_fn_d_i16_rem, calc_fn_d_i32_rem, calc_fn_d_i64_rem, calc_fn_d_f_rem, - calc_fn_d_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE + {NULL, NULL, calc_fn_i8_i8_rem, calc_fn_i8_i16_rem, calc_fn_i8_i32_rem, calc_fn_i8_i64_rem, calc_fn_i8_f_rem, calc_fn_i8_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT + {NULL, NULL, calc_fn_i16_i8_rem, calc_fn_i16_i16_rem, calc_fn_i16_i32_rem, calc_fn_i16_i64_rem, calc_fn_i16_f_rem, calc_fn_i16_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT + {NULL, NULL, calc_fn_i32_i8_rem, calc_fn_i32_i16_rem, calc_fn_i32_i32_rem, calc_fn_i32_i64_rem, calc_fn_i32_f_rem, calc_fn_i32_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_INT + {NULL, NULL, calc_fn_i64_i8_rem, calc_fn_i64_i16_rem, calc_fn_i64_i32_rem, calc_fn_i64_i64_rem, calc_fn_i64_f_rem, calc_fn_i64_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT + {NULL, NULL, calc_fn_f_i8_rem, calc_fn_f_i16_rem, calc_fn_f_i32_rem, calc_fn_f_i64_rem, calc_fn_f_f_rem, calc_fn_f_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT + {NULL, NULL, calc_fn_d_i8_rem, calc_fn_d_i16_rem, calc_fn_d_i32_rem, calc_fn_d_i64_rem, calc_fn_d_f_rem, calc_fn_d_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE }; //////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index ecb21db4de..7ebd43cd19 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -87,12 +87,26 @@ void taos_init_imp() { tsReadGlobalConfig(); tsPrintGlobalConfig(); - tscTrace("starting to initialize TAOS client ..."); tscTrace("Local IP address is:%s", tsLocalIp); } +#ifdef CLUSTER + tscMgmtIpList.numOfIps = 2; + strcpy(tscMgmtIpList.ipstr[0], tsMasterIp); + tscMgmtIpList.ip[0] = inet_addr(tsMasterIp); + + strcpy(tscMgmtIpList.ipstr[1], tsMasterIp); + tscMgmtIpList.ip[1] = inet_addr(tsMasterIp); + + if (tsSecondIp[0]) { + tscMgmtIpList.numOfIps = 3; + strcpy(tscMgmtIpList.ipstr[2], tsSecondIp); + tscMgmtIpList.ip[2] = inet_addr(tsSecondIp); + } +#endif + tscInitMsgs(); slaveIndex = rand(); int queueSize = tsMaxVnodeConnections + tsMaxMeterConnections + tsMaxMgmtConnections + tsMaxMgmtConnections; @@ -106,6 +120,10 @@ void taos_init_imp() { if (tscNumOfThreads < 2) tscNumOfThreads = 2; tscQhandle = taosInitScheduler(queueSize, tscNumOfThreads, "tsc"); + if (NULL == tscQhandle) { + tscError("failed to init scheduler"); + return; + } memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localIp = tsLocalIp; @@ -126,7 +144,14 @@ void taos_init_imp() { return; } - for (int i = 0; i < tscNumOfThreads; ++i) taosOpenRpcChann(pVnodeConn, i, rpcInit.sessionsPerChann); + for (int i = 0; i < tscNumOfThreads; ++i) { + int retVal = taosOpenRpcChann(pVnodeConn, i, rpcInit.sessionsPerChann); + if (0 != retVal) { + tError("TSC-vnode, failed to open rpc chann"); + taosCloseRpc(pVnodeConn); + return; + } + } memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localIp = tsLocalIp; @@ -148,10 +173,9 @@ void taos_init_imp() { } tscTmr = taosTmrInit(tsMaxMgmtConnections * 2, 200, 60000, "TSC"); - if (tscEmbedded == 0) { - taosTmrReset(tscCheckDiskUsage, 10, NULL, tscTmr, &tscCheckDiskUsageTmr); + if(0 == tscEmbedded){ + taosTmrReset(tscCheckDiskUsage, 10, NULL, tscTmr, &tscCheckDiskUsageTmr); } - int64_t refreshTime = tsMetricMetaKeepTimer < tsMeterMetaKeepTimer ? tsMetricMetaKeepTimer : tsMeterMetaKeepTimer; refreshTime = refreshTime > 2 ? 2 : refreshTime; refreshTime = refreshTime < 1 ? 1 : refreshTime; @@ -188,6 +212,7 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) { tsCfgStatusStr[cfg_configDir->cfgStatus], (char *)cfg_configDir->ptr); } break; + case TSDB_OPTION_SHELL_ACTIVITY_TIMER: if (cfg_activetimer && cfg_activetimer->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { tsShellActivityTimer = atoi((char *)arg); @@ -200,6 +225,7 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) { tsCfgStatusStr[cfg_activetimer->cfgStatus], (int32_t *)cfg_activetimer->ptr); } break; + case TSDB_OPTION_LOCALE: { // set locale pStr = (char *)arg; @@ -223,8 +249,7 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) { if (locale != NULL) { tscPrint("locale set, prev locale:%s, new locale:%s", tsLocale, locale); cfg_locale->cfgStatus = TSDB_CFG_CSTATUS_OPTION; - } else { - /* set the user-specified localed failed, use default LC_CTYPE as current locale */ + } else { // set the user-specified localed failed, use default LC_CTYPE as current locale locale = setlocale(LC_CTYPE, tsLocale); tscPrint("failed to set locale:%s, current locale:%s", pStr, tsLocale); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 4fb6b655af..ec5c980612 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -17,11 +17,13 @@ #include #include +#include "os.h" #include "ihash.h" #include "taosmsg.h" #include "tcache.h" #include "tkey.h" #include "tmd5.h" +#include "tscJoinProcess.h" #include "tscProfile.h" #include "tscSecondaryMerge.h" #include "tscUtil.h" @@ -32,45 +34,118 @@ /* * the detailed information regarding metric meta key is: - * fullmetername + '.' + querycond + '.' + [tagId1, tagId2,...] + '.' + group_orderType + * fullmetername + '.' + tagQueryCond + '.' + tableNameCond + '.' + joinCond + + * '.' + relation + '.' + [tagId1, tagId2,...] + '.' + group_orderType * - * if querycond is null, its format is: - * fullmetername + '.' + '(nil)' + '.' + [tagId1, tagId2,...] + '.' + group_orderType + * if querycond/tablenameCond/joinCond is null, its format is: + * fullmetername + '.' + '(nil)' + '.' + '(nil)' + relation + '.' + [tagId1, + * tagId2,...] + '.' + group_orderType */ -void tscGetMetricMetaCacheKey(SSqlCmd* pCmd, char* keyStr) { - char* pTagCondStr = NULL; - const int32_t RESERVED_SIZE = 100; +void tscGetMetricMetaCacheKey(SSqlCmd* pCmd, char* str, uint64_t uid) { + int32_t index = -1; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoByUid(pCmd, uid, &index); + int32_t len = 0; char tagIdBuf[128] = {0}; - int32_t offset = 0; - for (int32_t i = 0; i < pCmd->numOfReqTags; ++i) { - offset += sprintf(&tagIdBuf[offset], "%d,", pCmd->tagColumnIndex[i]); + for (int32_t i = 0; i < pMeterMetaInfo->numOfTags; ++i) { + len += sprintf(&tagIdBuf[len], "%d,", pMeterMetaInfo->tagColumnIndex[i]); } - assert(offset < tListLen(tagIdBuf)); - size_t len = strlen(pCmd->name); + STagCond* pTagCond = &pCmd->tagCond; + assert(len < tListLen(tagIdBuf)); - /* for too long key, we use the md5 to generated the key for local cache */ - if (pCmd->tagCond.len >= TSDB_MAX_TAGS_LEN - RESERVED_SIZE - offset) { + const int32_t maxKeySize = TSDB_MAX_TAGS_LEN; // allowed max key size + char* tmp = calloc(1, TSDB_MAX_SQL_LEN); + + SCond* cond = tsGetMetricQueryCondPos(pTagCond, uid); + + char join[512] = {0}; + if (pTagCond->joinInfo.hasJoin) { + sprintf(join, "%s,%s", pTagCond->joinInfo.left.meterId, pTagCond->joinInfo.right.meterId); + } + + int32_t keyLen = + snprintf(tmp, TSDB_MAX_SQL_LEN, "%s,%s,%s,%d,%s,[%s],%d", pMeterMetaInfo->name, + (cond != NULL ? cond->cond.z : NULL), pTagCond->tbnameCond.cond.n > 0 ? pTagCond->tbnameCond.cond.z : NULL, + pTagCond->relType, join, tagIdBuf, pCmd->groupbyExpr.orderType); + + assert(keyLen <= TSDB_MAX_SQL_LEN); + + if (keyLen < maxKeySize) { + strcpy(str, tmp); + } else { // using md5 to hash MD5_CTX ctx; MD5Init(&ctx); - MD5Update(&ctx, (uint8_t*)tsGetMetricQueryCondPos(&pCmd->tagCond), pCmd->tagCond.len); - MD5Final(&ctx); - pTagCondStr = base64_encode(ctx.digest, tListLen(ctx.digest)); - } else if (pCmd->tagCond.len + len + offset <= TSDB_MAX_TAGS_LEN && pCmd->tagCond.len > 0) { - pTagCondStr = strdup(tsGetMetricQueryCondPos(&pCmd->tagCond)); + MD5Update(&ctx, (uint8_t*) tmp, keyLen); + char* pStr = base64_encode(ctx.digest, tListLen(ctx.digest)); + strcpy(str, pStr); + } + + free(tmp); +} + +SCond* tsGetMetricQueryCondPos(STagCond* pTagCond, uint64_t uid) { + for (int32_t i = 0; i < TSDB_MAX_JOIN_TABLE_NUM; ++i) { + if (uid == pTagCond->cond[i].uid) { + return &pTagCond->cond[i]; + } + } + + return NULL; +} + +void tsSetMetricQueryCond(STagCond* pTagCond, uint64_t uid, const char* str) { + size_t len = strlen(str); + if (len == 0) { + return; } - int32_t keyLen = sprintf(keyStr, "%s.%s.[%s].%d", pCmd->name, pTagCondStr, tagIdBuf, pCmd->groupbyExpr.orderType); + SCond* pDest = &pTagCond->cond[pTagCond->numOfTagCond]; + pDest->uid = uid; + pDest->cond = SStringCreate(str); + + pTagCond->numOfTagCond += 1; +} + +bool tscQueryOnMetric(SSqlCmd* pCmd) { + return ((pCmd->type & TSDB_QUERY_TYPE_STABLE_QUERY) == TSDB_QUERY_TYPE_STABLE_QUERY) && + (pCmd->msgType == TSDB_MSG_TYPE_QUERY); +} + +bool tscQueryMetricTags(SSqlCmd* pCmd) { + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + if (tscSqlExprGet(pCmd, i)->functionId != TSDB_FUNC_TAGPRJ) { + return false; + } + } - free(pTagCondStr); - assert(keyLen <= TSDB_MAX_TAGS_LEN); + return true; } -char* tsGetMetricQueryCondPos(STagCond* pTagCond) { return pTagCond->pData; } +bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd) { + bool hasTags = false; + int32_t numOfSelectivity = 0; + + for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { + int32_t functId = tscSqlExprGet(pCmd, i)->functionId; + if (functId == TSDB_FUNC_TAG_DUMMY) { + hasTags = true; + continue; + } + + if ((aAggs[functId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + numOfSelectivity++; + } + } + + if (numOfSelectivity > 0 && hasTags) { + return true; + } + + return false; +} -bool tscQueryOnMetric(SSqlCmd* pCmd) { return UTIL_METER_IS_METRIC(pCmd) && pCmd->msgType == TSDB_MSG_TYPE_QUERY; } void tscGetDBInfoFromMeterId(char* meterId, char* db) { char* st = strstr(meterId, TS_PATH_DELIMITER); @@ -121,47 +196,57 @@ SMeterSidExtInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) { return (SMeterSidExtInfo*)(pSidList->pSidExtInfoList[idx] + (char*)pSidList); } -bool tscIsTwoStageMergeMetricQuery(SSqlObj* pSql) { - assert(pSql != NULL); +bool tscIsTwoStageMergeMetricQuery(SSqlCmd* pCmd) { + assert(pCmd != NULL); - SSqlCmd* pCmd = &pSql->cmd; - if (pCmd->pMeterMeta == NULL) { + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + if (pMeterMetaInfo == NULL || pMeterMetaInfo->pMetricMeta == NULL) { + return false; + } + + // for projection query, iterate all qualified vnodes sequentially + if (tscProjectionQueryOnMetric(pCmd)) { return false; } - if (pCmd->vnodeIdx == 0 && pCmd->command == TSDB_SQL_SELECT && (tscSqlExprGet(pCmd, 0)->sqlFuncId != TSDB_FUNC_PRJ)) { - return UTIL_METER_IS_METRIC(pCmd); + if (((pCmd->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) != TSDB_QUERY_TYPE_STABLE_SUBQUERY) && + pCmd->command == TSDB_SQL_SELECT) { + return UTIL_METER_IS_METRIC(pMeterMetaInfo); } return false; } -bool tscProjectionQueryOnMetric(SSqlObj* pSql) { - assert(pSql != NULL); +bool tscProjectionQueryOnMetric(SSqlCmd* pCmd) { + assert(pCmd != NULL); - SSqlCmd* pCmd = &pSql->cmd; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); /* * In following cases, return false for project query on metric * 1. failed to get metermeta from server; 2. not a metric; 3. limit 0; 4. show query, instead of a select query */ - if (pCmd->pMeterMeta == NULL || !UTIL_METER_IS_METRIC(pCmd) || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || - pCmd->exprsInfo.numOfExprs == 0) { + if (pMeterMetaInfo == NULL || !UTIL_METER_IS_METRIC(pMeterMetaInfo) || + pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pCmd->exprsInfo.numOfExprs == 0) { return false; } - /* - * Note:if there is COLPRJ_FUNCTION, only TAGPRJ_FUNCTION is allowed simultaneous - * for interp query, the query routine will action the same as projection query on metric - */ + // only query on tag, not a projection query + if (tscQueryMetricTags(pCmd)) { + return false; + } + + //for project query, only the following two function is allowed for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(&pSql->cmd, i); - if (pExpr->sqlFuncId == TSDB_FUNC_PRJ) { - return true; + SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); + int32_t functionId = pExpr->functionId; + if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && + functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TS) { + return false; } } - return false; + return true; } bool tscIsPointInterpQuery(SSqlCmd* pCmd) { @@ -171,7 +256,7 @@ bool tscIsPointInterpQuery(SSqlCmd* pCmd) { return false; } - int32_t functionId = pExpr->sqlFuncId; + int32_t functionId = pExpr->functionId; if (functionId == TSDB_FUNC_TAG) { continue; } @@ -180,12 +265,23 @@ bool tscIsPointInterpQuery(SSqlCmd* pCmd) { return false; } } - return true; } -bool tscIsFirstProjQueryOnMetric(SSqlObj* pSql) { - return (tscProjectionQueryOnMetric(pSql) && (pSql->cmd.vnodeIdx == 0)); +bool tscIsTWAQuery(SSqlCmd* pCmd) { + for(int32_t i = 0; i < pCmd->exprsInfo.numOfExprs; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); + if (pExpr == NULL) { + continue; + } + + int32_t functionId = pExpr->functionId; + if (functionId == TSDB_FUNC_TWA) { + return true; + } + } + + return false; } void tscClearInterpInfo(SSqlCmd* pCmd) { @@ -197,16 +293,12 @@ void tscClearInterpInfo(SSqlCmd* pCmd) { memset(pCmd->defaultVal, 0, sizeof(pCmd->defaultVal)); } -void tscClearSqlMetaInfo(SSqlCmd* pCmd) { - /* remove the metermeta/metricmeta in cache */ - taosRemoveDataFromCache(tscCacheHandle, (void**)&(pCmd->pMeterMeta), false); - taosRemoveDataFromCache(tscCacheHandle, (void**)&(pCmd->pMetricMeta), false); -} - void tscClearSqlMetaInfoForce(SSqlCmd* pCmd) { /* remove the metermeta/metricmeta in cache */ - taosRemoveDataFromCache(tscCacheHandle, (void**)&(pCmd->pMeterMeta), true); - taosRemoveDataFromCache(tscCacheHandle, (void**)&(pCmd->pMetricMeta), true); + // taosRemoveDataFromCache(tscCacheHandle, (void**)&(pCmd->pMeterMeta), + // true); + // taosRemoveDataFromCache(tscCacheHandle, (void**)&(pCmd->pMetricMeta), + // true); } int32_t tscCreateResPointerInfo(SSqlCmd* pCmd, SSqlRes* pRes) { @@ -251,21 +343,28 @@ void tscDestroyResPointerInfo(SSqlRes* pRes) { pRes->bytes = NULL; } -void tscfreeSqlCmdData(SSqlCmd* pCmd) { +void tscFreeSqlCmdData(SSqlCmd* pCmd) { pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); tscTagCondRelease(&pCmd->tagCond); - tscClearFieldInfo(pCmd); + tscClearFieldInfo(&pCmd->fieldsInfo); tfree(pCmd->exprsInfo.pExprs); memset(&pCmd->exprsInfo, 0, sizeof(pCmd->exprsInfo)); - tfree(pCmd->colList.pColList); + tscColumnBaseInfoDestroy(&pCmd->colList); memset(&pCmd->colList, 0, sizeof(pCmd->colList)); + + if (pCmd->tsBuf != NULL) { + tsBufDestory(pCmd->tsBuf); + pCmd->tsBuf = NULL; + } } void tscFreeSqlObjPartial(SSqlObj* pSql) { - if (pSql == NULL || pSql->signature != pSql) return; + if (pSql == NULL || pSql->signature != pSql) { + return; + } SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; @@ -273,10 +372,13 @@ void tscFreeSqlObjPartial(SSqlObj* pSql) { STscObj* pObj = pSql->pTscObj; int32_t cmd = pCmd->command; - if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_METRIC || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { + if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_METRIC || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || + cmd == TSDB_SQL_METRIC_JOIN_RETRIEVE) { tscRemoveFromSqlList(pSql); } + pCmd->command = -1; + // pSql->sqlstr will be used by tscBuildQueryStreamDesc pthread_mutex_lock(&pObj->mutex); tfree(pSql->sqlstr); @@ -295,9 +397,10 @@ void tscFreeSqlObjPartial(SSqlObj* pSql) { tfree(pSql->pSubs); pSql->numOfSubs = 0; tscDestroyResPointerInfo(pRes); + tfree(pSql->res.pColumnIndex); - tscfreeSqlCmdData(&pSql->cmd); - tscClearSqlMetaInfo(pCmd); + tscFreeSqlCmdData(pCmd); + tscRemoveAllMeterMetaInfo(pCmd, false); } void tscFreeSqlObj(SSqlObj* pSql) { @@ -308,10 +411,9 @@ void tscFreeSqlObj(SSqlObj* pSql) { pSql->signature = NULL; pSql->fp = NULL; - SSqlCmd* pCmd = &pSql->cmd; - memset(pCmd->payload, 0, (size_t)tsRpcHeadSize); + memset(pCmd->payload, 0, (size_t)pCmd->allocSize); tfree(pCmd->payload); pCmd->allocSize = 0; @@ -330,7 +432,6 @@ void tscFreeSqlObj(SSqlObj* pSql) { tsem_destroy(&pSql->rspSem); tsem_destroy(&pSql->emptyRspSem); } - free(pSql); } @@ -338,8 +439,7 @@ STableDataBlocks* tscCreateDataBlock(int32_t size) { STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); dataBuf->nAllocSize = (uint32_t)size; dataBuf->pData = calloc(1, dataBuf->nAllocSize); - - dataBuf->tsSource = -1; + dataBuf->ordered = true; dataBuf->prevTS = INT64_MIN; return dataBuf; } @@ -350,19 +450,62 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { } tfree(pDataBlock->pData); + tfree(pDataBlock->params); tfree(pDataBlock); } +SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, uint32_t offset) { + uint32_t needed = pDataBlock->numOfParams + 1; + if (needed > pDataBlock->numOfAllocedParams) { + needed *= 2; + void* tmp = realloc(pDataBlock->params, needed * sizeof(SParamInfo)); + if (tmp == NULL) { + return NULL; + } + pDataBlock->params = (SParamInfo*)tmp; + pDataBlock->numOfAllocedParams = needed; + } + + SParamInfo* param = pDataBlock->params + pDataBlock->numOfParams; + param->idx = -1; + param->type = type; + param->timePrec = timePrec; + param->bytes = bytes; + param->offset = offset; + + ++pDataBlock->numOfParams; + return param; +} + SDataBlockList* tscCreateBlockArrayList() { const int32_t DEFAULT_INITIAL_NUM_OF_BLOCK = 16; SDataBlockList* pDataBlockArrayList = calloc(1, sizeof(SDataBlockList)); + if (pDataBlockArrayList == NULL) { + return NULL; + } pDataBlockArrayList->nAlloc = DEFAULT_INITIAL_NUM_OF_BLOCK; pDataBlockArrayList->pData = calloc(1, POINTER_BYTES * pDataBlockArrayList->nAlloc); + if (pDataBlockArrayList->pData == NULL) { + free(pDataBlockArrayList); + return NULL; + } return pDataBlockArrayList; } +void tscAppendDataBlock(SDataBlockList *pList, STableDataBlocks *pBlocks) { + if (pList->nSize >= pList->nAlloc) { + pList->nAlloc = pList->nAlloc << 1; + pList->pData = realloc(pList->pData, sizeof(void *) * (size_t)pList->nAlloc); + + // reset allocated memory + memset(pList->pData + pList->nSize, 0, sizeof(void *) * (pList->nAlloc - pList->nSize)); + } + + pList->pData[pList->nSize++] = pBlocks; +} + void* tscDestroyBlockArrayList(SDataBlockList* pList) { if (pList == NULL) { return NULL; @@ -382,14 +525,15 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { SSqlCmd* pCmd = &pSql->cmd; pCmd->count = pDataBlock->numOfMeters; - strncpy(pCmd->name, pDataBlock->meterId, TSDB_METER_ID_LEN); + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + strcpy(pMeterMetaInfo->name, pDataBlock->meterId); /* * the submit message consists of : [RPC header|message body|digest] * the dataBlock only includes the RPC Header buffer and actual submit messsage body, space for digest needs * additional space. */ - int ret = tscAllocPayloadWithSize(pCmd, pDataBlock->nAllocSize + sizeof(STaosDigest)); + int ret = tscAllocPayload(pCmd, pDataBlock->nAllocSize + sizeof(STaosDigest)); if (TSDB_CODE_SUCCESS != ret) return ret; memcpy(pCmd->payload, pDataBlock->pData, pDataBlock->nAllocSize); @@ -400,7 +544,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize; assert(pCmd->allocSize >= pCmd->payloadLen + tsRpcHeadSize + sizeof(STaosDigest)); - return tscGetMeterMeta(pSql, pCmd->name); + return tscGetMeterMeta(pSql, pMeterMetaInfo->name, 0); } void tscFreeUnusedDataBlocks(SDataBlockList* pList) { @@ -413,10 +557,12 @@ void tscFreeUnusedDataBlocks(SDataBlockList* pList) { } STableDataBlocks* tscCreateDataBlockEx(size_t size, int32_t rowSize, int32_t startOffset, char* name) { - STableDataBlocks* dataBuf = tscCreateDataBlock(size); + STableDataBlocks *dataBuf = tscCreateDataBlock(size); dataBuf->rowSize = rowSize; dataBuf->size = startOffset; + dataBuf->tsSource = -1; + strncpy(dataBuf->meterId, name, TSDB_METER_ID_LEN); return dataBuf; } @@ -431,7 +577,7 @@ STableDataBlocks* tscGetDataBlockFromList(void* pHashList, SDataBlockList* pData } if (dataBuf == NULL) { - dataBuf = tscCreateDataBlockEx((size_t)size, rowSize, startOffset, tableId); + dataBuf = tscCreateDataBlockEx((size_t) size, rowSize, startOffset, tableId); dataBuf = *(STableDataBlocks**)taosAddIntHash(pHashList, id, (char*)&dataBuf); tscAppendDataBlock(pDataBlockList, dataBuf); } @@ -439,13 +585,15 @@ STableDataBlocks* tscGetDataBlockFromList(void* pHashList, SDataBlockList* pData return dataBuf; } -void tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockList) { - SSqlCmd* pCmd = &pSql->cmd; - void* pVnodeDataBlockHashList = taosInitIntHash(8, sizeof(void*), taosHashInt); +int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockList) { + SSqlCmd* pCmd = &pSql->cmd; + + void* pVnodeDataBlockHashList = taosInitIntHash(8, POINTER_BYTES, taosHashInt); SDataBlockList* pVnodeDataBlockList = tscCreateBlockArrayList(); for (int32_t i = 0; i < pTableDataBlockList->nSize; ++i) { STableDataBlocks* pOneTableBlock = pTableDataBlockList->pData[i]; + STableDataBlocks* dataBuf = tscGetDataBlockFromList(pVnodeDataBlockHashList, pVnodeDataBlockList, pOneTableBlock->vgid, TSDB_PAYLOAD_SIZE, tsInsertHeadSize, 0, pOneTableBlock->meterId); @@ -460,8 +608,14 @@ void tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockList) if (tmp != NULL) { dataBuf->pData = tmp; memset(dataBuf->pData + dataBuf->size, 0, dataBuf->nAllocSize - dataBuf->size); - } else { - // to do handle error + } else { // failed to allocate memory, free already allocated memory and return error code + tscError("%p failed to allocate memory for merging submit block, size:%d", pSql, dataBuf->nAllocSize); + + taosCleanUpIntHash(pVnodeDataBlockHashList); + tfree(dataBuf->pData); + tscDestroyBlockArrayList(pVnodeDataBlockList); + + return TSDB_CODE_CLI_OUT_OF_MEMORY; } } @@ -489,6 +643,8 @@ void tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockList) tscFreeUnusedDataBlocks(pCmd->pDataBlocks); taosCleanUpIntHash(pVnodeDataBlockHashList); + + return TSDB_CODE_SUCCESS; } void tscCloseTscObj(STscObj* pObj) { @@ -505,33 +661,25 @@ void tscCloseTscObj(STscObj* pObj) { } bool tscIsInsertOrImportData(char* sqlstr) { - SSQLToken t0 = {0}; - while (1) { - t0.n = tSQLGetToken(sqlstr, &t0.type); - if (t0.type != TK_SPACE) { - break; - } - - sqlstr += t0.n; - } - + int32_t index = 0; + SSQLToken t0 = tStrGetToken(sqlstr, &index, false, 0, NULL); return t0.type == TK_INSERT || t0.type == TK_IMPORT; } -int tscAllocPayloadWithSize(SSqlCmd* pCmd, int size) { +int tscAllocPayload(SSqlCmd* pCmd, int size) { assert(size > 0); if (pCmd->payload == NULL) { assert(pCmd->allocSize == 0); - pCmd->payload = (char*)calloc(1, size); + pCmd->payload = (char*)malloc(size); if (pCmd->payload == NULL) return TSDB_CODE_CLI_OUT_OF_MEMORY; - pCmd->allocSize = size; } else { if (pCmd->allocSize < size) { - pCmd->payload = realloc(pCmd->payload, size); - if (pCmd->payload == NULL) return TSDB_CODE_CLI_OUT_OF_MEMORY; + char* b = realloc(pCmd->payload, size); + if (b == NULL) return TSDB_CODE_CLI_OUT_OF_MEMORY; + pCmd->payload = b; pCmd->allocSize = size; } } @@ -563,6 +711,8 @@ static void ensureSpace(SFieldInfo* pFieldInfo, int32_t size) { pFieldInfo->pOffset = realloc(pFieldInfo->pOffset, newSize * sizeof(int16_t)); memset(&pFieldInfo->pOffset[oldSize], 0, inc * sizeof(int16_t)); + pFieldInfo->pVisibleCols = realloc(pFieldInfo->pVisibleCols, newSize * sizeof(bool)); + pFieldInfo->numOfAlloc = newSize; } } @@ -594,15 +744,38 @@ void tscFieldInfoSetValFromField(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIE evic(pFieldInfo, index); memcpy(&pFieldInfo->pFields[index], pField, sizeof(TAOS_FIELD)); + pFieldInfo->pVisibleCols[index] = true; + pFieldInfo->numOfOutputCols++; } +void tscFieldInfoUpdateVisible(SFieldInfo* pFieldInfo, int32_t index, bool visible) { + if (index < 0 || index > pFieldInfo->numOfOutputCols) { + return; + } + + bool oldVisible = pFieldInfo->pVisibleCols[index]; + pFieldInfo->pVisibleCols[index] = visible; + + if (oldVisible != visible) { + if (!visible) { + pFieldInfo->numOfHiddenCols += 1; + } else { + if (pFieldInfo->numOfHiddenCols > 0) { + pFieldInfo->numOfHiddenCols -= 1; + } + } + } +} + void tscFieldInfoSetValue(SFieldInfo* pFieldInfo, int32_t index, int8_t type, char* name, int16_t bytes) { ensureSpace(pFieldInfo, pFieldInfo->numOfOutputCols + 1); evic(pFieldInfo, index); TAOS_FIELD* pField = &pFieldInfo->pFields[index]; setValueImpl(pField, type, name, bytes); + + pFieldInfo->pVisibleCols[index] = true; pFieldInfo->numOfOutputCols++; } @@ -615,7 +788,7 @@ void tscFieldInfoCalOffset(SSqlCmd* pCmd) { } } -void tscFieldInfoRenewOffsetForInterResult(SSqlCmd* pCmd) { +void tscFieldInfoUpdateOffset(SSqlCmd* pCmd) { SFieldInfo* pFieldInfo = &pCmd->fieldsInfo; if (pFieldInfo->numOfOutputCols == 0) { return; @@ -632,18 +805,32 @@ void tscFieldInfoRenewOffsetForInterResult(SSqlCmd* pCmd) { } } -void tscFieldInfoClone(SFieldInfo* src, SFieldInfo* dst) { +void tscFieldInfoCopy(SFieldInfo* src, SFieldInfo* dst, const int32_t* indexList, int32_t size) { if (src == NULL) { return; } + if (size <= 0) { + *dst = *src; + tscFieldInfoCopyAll(src, dst); + } else { // only copy the required column + for (int32_t i = 0; i < size; ++i) { + assert(indexList[i] >= 0 && indexList[i] <= src->numOfOutputCols); + tscFieldInfoSetValFromField(dst, i, &src->pFields[indexList[i]]); + } + } +} + +void tscFieldInfoCopyAll(SFieldInfo* src, SFieldInfo* dst) { *dst = *src; dst->pFields = malloc(sizeof(TAOS_FIELD) * dst->numOfAlloc); dst->pOffset = malloc(sizeof(short) * dst->numOfAlloc); + dst->pVisibleCols = malloc(sizeof(bool) * dst->numOfAlloc); memcpy(dst->pFields, src->pFields, sizeof(TAOS_FIELD) * dst->numOfOutputCols); memcpy(dst->pOffset, src->pOffset, sizeof(short) * dst->numOfOutputCols); + memcpy(dst->pVisibleCols, src->pVisibleCols, sizeof(bool) * dst->numOfOutputCols); } TAOS_FIELD* tscFieldInfoGetField(SSqlCmd* pCmd, int32_t index) { @@ -672,14 +859,16 @@ int32_t tscGetResRowLength(SSqlCmd* pCmd) { pFieldInfo->pFields[pFieldInfo->numOfOutputCols - 1].bytes; } -void tscClearFieldInfo(SSqlCmd* pCmd) { - if (pCmd == NULL) { +void tscClearFieldInfo(SFieldInfo* pFieldInfo) { + if (pFieldInfo == NULL) { return; } - tfree(pCmd->fieldsInfo.pOffset); - tfree(pCmd->fieldsInfo.pFields); - memset(&pCmd->fieldsInfo, 0, sizeof(pCmd->fieldsInfo)); + tfree(pFieldInfo->pOffset); + tfree(pFieldInfo->pFields); + tfree(pFieldInfo->pVisibleCols); + + memset(pFieldInfo, 0, sizeof(SFieldInfo)); } static void _exprCheckSpace(SSqlExprInfo* pExprInfo, int32_t size) { @@ -711,28 +900,45 @@ static void _exprEvic(SSqlExprInfo* pExprInfo, int32_t index) { } } -SSqlExpr* tscSqlExprInsert(SSqlCmd* pCmd, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, - int16_t size) { +SSqlExpr* tscSqlExprInsert(SSqlCmd* pCmd, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, int16_t interSize) { + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pColIndex->tableIndex); + SSqlExprInfo* pExprInfo = &pCmd->exprsInfo; - SSchema* pSchema = tsGetSchema(pCmd->pMeterMeta); _exprCheckSpace(pExprInfo, pExprInfo->numOfExprs + 1); _exprEvic(pExprInfo, index); SSqlExpr* pExpr = &pExprInfo->pExprs[index]; - pExpr->sqlFuncId = functionId; + pExpr->functionId = functionId; + int16_t numOfCols = pMeterMetaInfo->pMeterMeta->numOfColumns; - pExpr->colInfo.colIdx = srcColumnIndex; - if (srcColumnIndex == -1) { - pExpr->colInfo.colId = -1; + // set the correct column index + if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; + } else { + SSchema* pSchema = tsGetColumnSchema(pMeterMetaInfo->pMeterMeta, pColIndex->columnIndex); + pExpr->colInfo.colId = pSchema->colId; + } + + // tag columns require the column index revised. + if (pColIndex->columnIndex >= numOfCols) { + pColIndex->columnIndex -= numOfCols; + pExpr->colInfo.flag = TSDB_COL_TAG; } else { - pExpr->colInfo.colId = pSchema[srcColumnIndex].colId; + if (pColIndex->columnIndex != TSDB_TBNAME_COLUMN_INDEX) { + pExpr->colInfo.flag = TSDB_COL_NORMAL; + } else { + pExpr->colInfo.flag = TSDB_COL_TAG; + } } - pExpr->colInfo.isTag = false; + pExpr->colInfo.colIdx = pColIndex->columnIndex; pExpr->resType = type; pExpr->resBytes = size; + pExpr->interResBytes = interSize; + pExpr->uid = pMeterMetaInfo->pMeterMeta->uid; pExprInfo->numOfExprs++; return pExpr; @@ -740,17 +946,18 @@ SSqlExpr* tscSqlExprInsert(SSqlCmd* pCmd, int32_t index, int16_t functionId, int SSqlExpr* tscSqlExprUpdate(SSqlCmd* pCmd, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, int16_t size) { - SSqlExprInfo* pExprInfo = &pCmd->exprsInfo; + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + SSqlExprInfo* pExprInfo = &pCmd->exprsInfo; if (index > pExprInfo->numOfExprs) { return NULL; } SSqlExpr* pExpr = &pExprInfo->pExprs[index]; - pExpr->sqlFuncId = functionId; + pExpr->functionId = functionId; pExpr->colInfo.colIdx = srcColumnIndex; - pExpr->colInfo.colId = tsGetSchemaColIdx(pCmd->pMeterMeta, srcColumnIndex)->colId; + pExpr->colInfo.colId = tsGetColumnSchema(pMeterMetaInfo->pMeterMeta, srcColumnIndex)->colId; pExpr->resType = type; pExpr->resBytes = size; @@ -758,14 +965,14 @@ SSqlExpr* tscSqlExprUpdate(SSqlCmd* pCmd, int32_t index, int16_t functionId, int return pExpr; } -void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes) { +void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex) { if (pExpr == NULL || argument == NULL || bytes == 0) { return; } // set parameter value // transfer to tVariant from byte data/no ascii data - tVariantCreateB(&pExpr->param[pExpr->numOfParams], argument, bytes, type); + tVariantCreateFromBinary(&pExpr->param[pExpr->numOfParams], argument, bytes, type); pExpr->numOfParams += 1; assert(pExpr->numOfParams <= 3); @@ -779,7 +986,7 @@ SSqlExpr* tscSqlExprGet(SSqlCmd* pCmd, int32_t index) { return &pCmd->exprsInfo.pExprs[index]; } -void tscSqlExprClone(SSqlExprInfo* src, SSqlExprInfo* dst) { +void tscSqlExprCopy(SSqlExprInfo* dst, const SSqlExprInfo* src, uint64_t tableuid) { if (src == NULL) { return; } @@ -787,8 +994,14 @@ void tscSqlExprClone(SSqlExprInfo* src, SSqlExprInfo* dst) { *dst = *src; dst->pExprs = malloc(sizeof(SSqlExpr) * dst->numOfAlloc); - memcpy(dst->pExprs, src->pExprs, sizeof(SSqlExpr) * dst->numOfExprs); + int16_t num = 0; + for (int32_t i = 0; i < src->numOfExprs; ++i) { + if (src->pExprs[i].uid == tableuid) { + dst->pExprs[num++] = src->pExprs[i]; + } + } + dst->numOfExprs = num; for (int32_t i = 0; i < dst->numOfExprs; ++i) { for (int32_t j = 0; j < src->pExprs[i].numOfParams; ++j) { tVariantAssign(&dst->pExprs[i].param[j], &src->pExprs[i].param[j]); @@ -796,7 +1009,14 @@ void tscSqlExprClone(SSqlExprInfo* src, SSqlExprInfo* dst) { } } -static void _cf_ensureSpace(SColumnsInfo* pcolList, int32_t size) { +static void clearVal(SColumnBase* pBase) { + memset(pBase, 0, sizeof(SColumnBase)); + + pBase->colIndex.tableIndex = -2; + pBase->colIndex.columnIndex = -2; +} + +static void _cf_ensureSpace(SColumnBaseInfo* pcolList, int32_t size) { if (pcolList->numOfAlloc < size) { int32_t oldSize = pcolList->numOfAlloc; @@ -818,41 +1038,58 @@ static void _cf_ensureSpace(SColumnsInfo* pcolList, int32_t size) { } } -static void _cf_evic(SColumnsInfo* pcolList, int32_t index) { +static void _cf_evic(SColumnBaseInfo* pcolList, int32_t index) { if (index < pcolList->numOfCols) { memmove(&pcolList->pColList[index + 1], &pcolList->pColList[index], sizeof(SColumnBase) * (pcolList->numOfCols - index)); - memset(&pcolList->pColList[index], 0, sizeof(SColumnBase)); + clearVal(&pcolList->pColList[index]); } } -SColumnBase* tscColumnInfoGet(SSqlCmd* pCmd, int32_t index) { - if (pCmd->colList.numOfCols < index) { +SColumnBase* tscColumnBaseInfoGet(SColumnBaseInfo* pColumnBaseInfo, int32_t index) { + if (pColumnBaseInfo == NULL || pColumnBaseInfo->numOfCols < index) { return NULL; } - return &pCmd->colList.pColList[index]; + return &pColumnBaseInfo->pColList[index]; +} + +void tscColumnBaseInfoUpdateTableIndex(SColumnBaseInfo* pColList, int16_t tableIndex) { + for (int32_t i = 0; i < pColList->numOfCols; ++i) { + pColList->pColList[i].colIndex.tableIndex = tableIndex; + } } -SColumnBase* tscColumnInfoInsert(SSqlCmd* pCmd, int32_t colIndex) { - SColumnsInfo* pcolList = &pCmd->colList; +// todo refactor +SColumnBase* tscColumnBaseInfoInsert(SSqlCmd* pCmd, SColumnIndex* pColIndex) { + SColumnBaseInfo* pcolList = &pCmd->colList; - if (colIndex < 0) { - /* ignore the tbname column to be inserted into source list */ + // ignore the tbname column to be inserted into source list + if (pColIndex->columnIndex < 0) { return NULL; } + int16_t col = pColIndex->columnIndex; + int32_t i = 0; - while (i < pcolList->numOfCols && pcolList->pColList[i].colIndex < colIndex) { - i++; + while (i < pcolList->numOfCols) { + if (pcolList->pColList[i].colIndex.columnIndex < col) { + i++; + } else if (pcolList->pColList[i].colIndex.tableIndex < pColIndex->tableIndex) { + i++; + } else { + break; + } } - if ((i < pcolList->numOfCols && pcolList->pColList[i].colIndex > colIndex) || (i >= pcolList->numOfCols)) { + SColumnIndex* pIndex = &pcolList->pColList[i].colIndex; + if ((i < pcolList->numOfCols && (pIndex->columnIndex > col || pIndex->tableIndex != pColIndex->tableIndex)) || + (i >= pcolList->numOfCols)) { _cf_ensureSpace(pcolList, pcolList->numOfCols + 1); _cf_evic(pcolList, i); - pcolList->pColList[i].colIndex = (int16_t)colIndex; + pcolList->pColList[i].colIndex = *pColIndex; pcolList->numOfCols++; pCmd->numOfCols++; } @@ -860,18 +1097,94 @@ SColumnBase* tscColumnInfoInsert(SSqlCmd* pCmd, int32_t colIndex) { return &pcolList->pColList[i]; } -void tscColumnInfoClone(SColumnsInfo* src, SColumnsInfo* dst) { +void tscColumnFilterInfoCopy(SColumnFilterInfo* dst, const SColumnFilterInfo* src) { + assert (src != NULL && dst != NULL); + + assert(src->filterOnBinary == 0 || src->filterOnBinary == 1); + if (src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID) { + assert(0); + } + + *dst = *src; + if (dst->filterOnBinary) { + size_t len = (size_t) dst->len + 1; + dst->pz = calloc(1, len); + memcpy((char*) dst->pz, (char*) src->pz, (size_t) len); + } +} + +void tscColumnBaseCopy(SColumnBase* dst, const SColumnBase* src) { + assert (src != NULL && dst != NULL); + + *dst = *src; + + if (src->numOfFilters > 0) { + dst->filterInfo = calloc(1, src->numOfFilters * sizeof(SColumnFilterInfo)); + + for (int32_t j = 0; j < src->numOfFilters; ++j) { + tscColumnFilterInfoCopy(&dst->filterInfo[j], &src->filterInfo[j]); + } + } else { + assert(src->filterInfo == NULL); + } +} + +void tscColumnBaseInfoCopy(SColumnBaseInfo* dst, const SColumnBaseInfo* src, int16_t tableIndex) { if (src == NULL) { return; } *dst = *src; + dst->pColList = calloc(1, sizeof(SColumnBase) * dst->numOfAlloc); + + int16_t num = 0; + for (int32_t i = 0; i < src->numOfCols; ++i) { + if (src->pColList[i].colIndex.tableIndex == tableIndex || tableIndex < 0) { + dst->pColList[num] = src->pColList[i]; + + if (dst->pColList[num].numOfFilters > 0) { + dst->pColList[num].filterInfo = calloc(1, dst->pColList[num].numOfFilters * sizeof(SColumnFilterInfo)); + + for (int32_t j = 0; j < dst->pColList[num].numOfFilters; ++j) { + tscColumnFilterInfoCopy(&dst->pColList[num].filterInfo[j], &src->pColList[i].filterInfo[j]); + } + } + + num += 1; + } + } - dst->pColList = malloc(sizeof(SColumnBase) * dst->numOfAlloc); - memcpy(dst->pColList, src->pColList, sizeof(SColumnBase) * dst->numOfCols); + dst->numOfCols = num; } -void tscColumnInfoReserve(SSqlCmd* pCmd, int32_t size) { _cf_ensureSpace(&pCmd->colList, size); } +void tscColumnBaseInfoDestroy(SColumnBaseInfo* pColumnBaseInfo) { + if (pColumnBaseInfo == NULL) { + return; + } + + assert(pColumnBaseInfo->numOfCols <= TSDB_MAX_COLUMNS); + + for (int32_t i = 0; i < pColumnBaseInfo->numOfCols; ++i) { + SColumnBase *pColBase = &(pColumnBaseInfo->pColList[i]); + + if (pColBase->numOfFilters > 0) { + for (int32_t j = 0; j < pColBase->numOfFilters; ++j) { + assert(pColBase->filterInfo[j].filterOnBinary == 0 || pColBase->filterInfo[j].filterOnBinary == 1); + + if (pColBase->filterInfo[j].filterOnBinary) { + tfree(pColBase->filterInfo[j].pz); + } + } + } + + tfree(pColBase->filterInfo); + } + + tfree(pColumnBaseInfo->pColList); +} + + +void tscColumnBaseInfoReserve(SColumnBaseInfo* pColumnBaseInfo, int32_t size) { _cf_ensureSpace(pColumnBaseInfo, size); } /* * 1. normal name, not a keyword or number @@ -887,7 +1200,6 @@ void tscColumnInfoReserve(SSqlCmd* pCmd, int32_t size) { _cf_ensureSpace(&pCmd-> * 'first_part.second_part' * */ - static int32_t validateQuoteToken(SSQLToken* pToken) { pToken->n = strdequote(pToken->z); strtrim(pToken->z); @@ -920,16 +1232,16 @@ int32_t tscValidateName(SSQLToken* pToken) { int len = tSQLGetToken(pToken->z, &pToken->type); // single token, validate it - if (len == pToken->n) { + if (len == pToken->n){ return validateQuoteToken(pToken); } else { - sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true); - if (sep == NULL) { - return TSDB_CODE_INVALID_SQL; - } + sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true); + if (sep == NULL) { + return TSDB_CODE_INVALID_SQL; + } return tscValidateName(pToken); - } + } } else { if (isNumber(pToken)) { return TSDB_CODE_INVALID_SQL; @@ -996,16 +1308,17 @@ void tscIncStreamExecutionCount(void* pStream) { } bool tscValidateColumnId(SSqlCmd* pCmd, int32_t colId) { - if (pCmd->pMeterMeta == NULL) { + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + if (pMeterMetaInfo->pMeterMeta == NULL) { return false; } - if (colId == -1 && UTIL_METER_IS_METRIC(pCmd)) { + if (colId == -1 && UTIL_METER_IS_METRIC(pMeterMetaInfo)) { return true; } - SSchema* pSchema = tsGetSchema(pCmd->pMeterMeta); - int32_t numOfTotal = pCmd->pMeterMeta->numOfTags + pCmd->pMeterMeta->numOfColumns; + SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + int32_t numOfTotal = pMeterMetaInfo->pMeterMeta->numOfTags + pMeterMetaInfo->pMeterMeta->numOfColumns; for (int32_t i = 0; i < numOfTotal; ++i) { if (pSchema[i].colId == colId) { @@ -1016,37 +1329,44 @@ bool tscValidateColumnId(SSqlCmd* pCmd, int32_t colId) { return false; } -void tscTagCondAssign(STagCond* pDst, STagCond* pSrc) { - if (pSrc->len == 0) { - memset(pDst, 0, sizeof(STagCond)); - return; +void tscTagCondCopy(STagCond* dest, const STagCond* src) { + memset(dest, 0, sizeof(STagCond)); + + SStringCopy(&dest->tbnameCond.cond, &src->tbnameCond.cond); + dest->tbnameCond.uid = src->tbnameCond.uid; + + memcpy(&dest->joinInfo, &src->joinInfo, sizeof(SJoinInfo)); + + for (int32_t i = 0; i < src->numOfTagCond; ++i) { + SStringCopy(&dest->cond[i].cond, &src->cond[i].cond); + dest->cond[i].uid = src->cond[i].uid; } - pDst->pData = strdup(pSrc->pData); - pDst->allocSize = pSrc->len + 1; - pDst->type = pSrc->type; - pDst->len = pSrc->len; + dest->relType = src->relType; + dest->numOfTagCond = src->numOfTagCond; } void tscTagCondRelease(STagCond* pCond) { - if (pCond->allocSize > 0) { - assert(pCond->pData != NULL); - tfree(pCond->pData); + SStringFree(&pCond->tbnameCond.cond); + + for (int32_t i = 0; i < pCond->numOfTagCond; ++i) { + SStringFree(&pCond->cond[i].cond); } memset(pCond, 0, sizeof(STagCond)); } void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SSqlCmd* pCmd) { - SSchema* pSchema = tsGetSchema(pCmd->pMeterMeta); + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); for (int32_t i = 0; i < pCmd->exprsInfo.numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); - pColInfo[i].functionId = pExpr->sqlFuncId; + pColInfo[i].functionId = pExpr->functionId; - if (pExpr->colInfo.isTag) { - SSchema* pTagSchema = tsGetTagSchema(pCmd->pMeterMeta); - int16_t actualTagIndex = pCmd->tagColumnIndex[pExpr->colInfo.colIdx]; + if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { + SSchema* pTagSchema = tsGetTagSchema(pMeterMetaInfo->pMeterMeta); + int16_t actualTagIndex = pMeterMetaInfo->tagColumnIndex[pExpr->colInfo.colIdx]; pColInfo[i].type = (actualTagIndex != -1) ? pTagSchema[actualTagIndex].type : TSDB_DATA_TYPE_BINARY; } else { @@ -1063,30 +1383,28 @@ void tscSetFreeHeatBeat(STscObj* pObj) { SSqlObj* pHeatBeat = pObj->pHb; assert(pHeatBeat == pHeatBeat->signature); - pHeatBeat->cmd.type = 1; // to denote the heart-beat timer close connection and free all allocated resources + // to denote the heart-beat timer close connection and free all allocated resources + pHeatBeat->cmd.type = TSDB_QUERY_TYPE_FREE_RESOURCE; } bool tscShouldFreeHeatBeat(SSqlObj* pHb) { assert(pHb == pHb->signature); - - return pHb->cmd.type == 1; + return pHb->cmd.type == TSDB_QUERY_TYPE_FREE_RESOURCE; } void tscCleanSqlCmd(SSqlCmd* pCmd) { - tscfreeSqlCmdData(pCmd); + tscFreeSqlCmdData(pCmd); + + assert(pCmd->pMeterInfo == NULL); - uint32_t allocSize = pCmd->allocSize; - char* allocPtr = pCmd->payload; - SMeterMeta* pMeterMeta = pCmd->pMeterMeta; - SMetricMeta* pMetricMeta = pCmd->pMetricMeta; + uint32_t allocSize = pCmd->allocSize; + char* allocPtr = pCmd->payload; memset(pCmd, 0, sizeof(SSqlCmd)); // restore values pCmd->allocSize = allocSize; pCmd->payload = allocPtr; - pCmd->pMeterMeta = pMeterMeta; - pCmd->pMetricMeta = pMetricMeta; } /* @@ -1143,8 +1461,339 @@ bool tscShouldFreeAsyncSqlObj(SSqlObj* pSql) { } } +SMeterMetaInfo* tscGetMeterMetaInfo(SSqlCmd* pCmd, int32_t index) { + if (pCmd == NULL || index >= pCmd->numOfTables || index < 0) { + return NULL; + } + + return pCmd->pMeterInfo[index]; +} + +SMeterMetaInfo* tscGetMeterMetaInfoByUid(SSqlCmd* pCmd, uint64_t uid, int32_t* index) { + int32_t k = -1; + for (int32_t i = 0; i < pCmd->numOfTables; ++i) { + if (pCmd->pMeterInfo[i]->pMeterMeta->uid == uid) { + k = i; + break; + } + } + + if (index != NULL) { + *index = k; + } + + return tscGetMeterMetaInfo(pCmd, k); +} + +SMeterMetaInfo* tscAddMeterMetaInfo(SSqlCmd* pCmd, const char* name, SMeterMeta* pMeterMeta, SMetricMeta* pMetricMeta, + int16_t numOfTags, int16_t* tags) { + void* pAlloc = realloc(pCmd->pMeterInfo, (pCmd->numOfTables + 1) * POINTER_BYTES); + if (pAlloc == NULL) { + return NULL; + } + + pCmd->pMeterInfo = pAlloc; + pCmd->pMeterInfo[pCmd->numOfTables] = calloc(1, sizeof(SMeterMetaInfo)); + + SMeterMetaInfo* pMeterMetaInfo = pCmd->pMeterInfo[pCmd->numOfTables]; + assert(pMeterMetaInfo != NULL); + + if (name != NULL) { + assert(strlen(name) <= TSDB_METER_ID_LEN); + strcpy(pMeterMetaInfo->name, name); + } + + pMeterMetaInfo->pMeterMeta = pMeterMeta; + pMeterMetaInfo->pMetricMeta = pMetricMeta; + pMeterMetaInfo->numOfTags = numOfTags; + + if (tags != NULL) { + memcpy(pMeterMetaInfo->tagColumnIndex, tags, sizeof(int16_t) * numOfTags); + } + + pCmd->numOfTables += 1; + + return pMeterMetaInfo; +} + +SMeterMetaInfo* tscAddEmptyMeterMetaInfo(SSqlCmd* pCmd) { return tscAddMeterMetaInfo(pCmd, NULL, NULL, NULL, 0, NULL); } + +void tscRemoveMeterMetaInfo(SSqlCmd* pCmd, int32_t index, bool removeFromCache) { + if (index < 0 || index >= pCmd->numOfTables) { + return; + } + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index); + + tscClearMeterMetaInfo(pMeterMetaInfo, removeFromCache); + free(pMeterMetaInfo); + + int32_t after = pCmd->numOfTables - index - 1; + if (after > 0) { + memmove(&pCmd->pMeterInfo[index], &pCmd->pMeterInfo[index + 1], after * sizeof(void*)); + } + + pCmd->numOfTables -= 1; +} + +void tscRemoveAllMeterMetaInfo(SSqlCmd* pCmd, bool removeFromCache) { + int64_t addr = offsetof(SSqlObj, cmd); + + tscTrace("%p deref the metric/meter meta in cache, numOfTables:%d", ((char*)pCmd - addr), pCmd->numOfTables); + + while (pCmd->numOfTables > 0) { + tscRemoveMeterMetaInfo(pCmd, pCmd->numOfTables - 1, removeFromCache); + } + + tfree(pCmd->pMeterInfo); +} + +void tscClearMeterMetaInfo(SMeterMetaInfo* pMeterMetaInfo, bool removeFromCache) { + if (pMeterMetaInfo == NULL) { + return; + } + + taosRemoveDataFromCache(tscCacheHandle, (void**)&(pMeterMetaInfo->pMeterMeta), removeFromCache); + taosRemoveDataFromCache(tscCacheHandle, (void**)&(pMeterMetaInfo->pMetricMeta), removeFromCache); +} + +void tscResetForNextRetrieve(SSqlRes* pRes) { + pRes->row = 0; + pRes->numOfRows = 0; +} + +SString SStringCreate(const char* str) { + size_t len = strlen(str); + + SString dest = {.n = len, .alloc = len + 1}; + dest.z = calloc(1, dest.alloc); + strcpy(dest.z, str); + + return dest; +} + +void SStringCopy(SString* pDest, const SString* pSrc) { + if (pSrc->n > 0) { + pDest->n = pSrc->n; + pDest->alloc = pDest->n + 1; // one additional space for null terminate + + pDest->z = calloc(1, pDest->alloc); + + memcpy(pDest->z, pSrc->z, pDest->n); + } else { + memset(pDest, 0, sizeof(SString)); + } +} + +void SStringFree(SString* pStr) { + if (pStr->alloc > 0) { + tfree(pStr->z); + pStr->alloc = 0; + } +} + +void SStringShrink(SString* pStr) { + if (pStr->alloc > (pStr->n + 1) && pStr->alloc > (pStr->n * 2)) { + pStr->z = realloc(pStr->z, pStr->n + 1); + assert(pStr->z != NULL); + + pStr->alloc = pStr->n + 1; + } +} + +int32_t SStringAlloc(SString* pStr, int32_t size) { + if (pStr->alloc >= size) { + return TSDB_CODE_SUCCESS; + } + + size = ALIGN8(size); + + char* tmp = NULL; + if (pStr->z != NULL) { + tmp = realloc(pStr->z, size); + memset(pStr->z + pStr->n, 0, size - pStr->n); + } else { + tmp = calloc(1, size); + } + + if (tmp == NULL) { +#ifdef WINDOWS + LPVOID lpMsgBuf; + FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, + GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language + (LPTSTR)&lpMsgBuf, 0, NULL); + tscTrace("failed to allocate memory, reason:%s", lpMsgBuf); + LocalFree(lpMsgBuf); +#else + char errmsg[256] = {0}; + strerror_r(errno, errmsg, tListLen(errmsg)); + tscTrace("failed to allocate memory, reason:%s", errmsg); +#endif + return TSDB_CODE_CLI_OUT_OF_MEMORY; + } + + pStr->z = tmp; + pStr->alloc = size; + + return TSDB_CODE_SUCCESS; +} + +#define MIN_ALLOC_SIZE 8 + +int32_t SStringEnsureRemain(SString* pStr, int32_t size) { + if (pStr->alloc - pStr->n > size) { + return TSDB_CODE_SUCCESS; + } + + // remain space is insufficient, allocate more spaces + int32_t inc = (size < MIN_ALLOC_SIZE) ? size : MIN_ALLOC_SIZE; + if (inc < (pStr->alloc >> 1)) { + inc = (pStr->alloc >> 1); + } + + // get the new size + int32_t newsize = pStr->alloc + inc; + + char* tmp = realloc(pStr->z, newsize); + if (tmp == NULL) { + +#ifdef WINDOWS + LPVOID lpMsgBuf; + FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, + GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language + (LPTSTR)&lpMsgBuf, 0, NULL); + tscTrace("failed to allocate memory, reason:%s", lpMsgBuf); + LocalFree(lpMsgBuf); +#else + char errmsg[256] = {0}; + strerror_r(errno, errmsg, tListLen(errmsg)); + tscTrace("failed to allocate memory, reason:%s", errmsg); +#endif + + return TSDB_CODE_CLI_OUT_OF_MEMORY; + } + + memset(tmp + pStr->n, 0, inc); + pStr->z = tmp; + + return TSDB_CODE_SUCCESS; +} + +SSqlObj* createSubqueryObj(SSqlObj* pSql, int32_t vnodeIndex, int16_t tableIndex, void (*fp)(), void* param, + SSqlObj* pPrevSql) { + SSqlCmd* pCmd = &pSql->cmd; + + SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); + if (pNew == NULL) { + tscError("%p new subquery failed, vnodeIdx:%d, tableIndex:%d", pSql, vnodeIndex, tableIndex); + return NULL; + } + + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + + pNew->sqlstr = strdup(pSql->sqlstr); + if (pNew->sqlstr == NULL) { + tscError("%p new subquery failed, vnodeIdx:%d, tableIndex:%d", pSql, vnodeIndex, tableIndex); + + free(pNew); + return NULL; + } + + memcpy(&pNew->cmd, pCmd, sizeof(SSqlCmd)); + + pNew->cmd.command = TSDB_SQL_SELECT; + pNew->cmd.payload = NULL; + pNew->cmd.allocSize = 0; + + pNew->cmd.pMeterInfo = NULL; + + pNew->cmd.colList.pColList = NULL; + pNew->cmd.colList.numOfAlloc = 0; + pNew->cmd.colList.numOfCols = 0; + + pNew->cmd.numOfTables = 0; + pNew->cmd.tsBuf = NULL; + + memset(&pNew->cmd.fieldsInfo, 0, sizeof(SFieldInfo)); + tscTagCondCopy(&pNew->cmd.tagCond, &pCmd->tagCond); + + if (tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) { + tscError("%p new subquery failed, vnodeIdx:%d, tableIndex:%d", pSql, vnodeIndex, tableIndex); + tscFreeSqlObj(pNew); + return NULL; + } + + tscColumnBaseInfoCopy(&pNew->cmd.colList, &pCmd->colList, (int16_t)tableIndex); + + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, tableIndex); + + // set the correct query type + if (pPrevSql != NULL) { + pNew->cmd.type = pPrevSql->cmd.type; + } else { + pNew->cmd.type |= TSDB_QUERY_TYPE_SUBQUERY; // it must be the subquery + } + + uint64_t uid = pMeterMetaInfo->pMeterMeta->uid; + tscSqlExprCopy(&pNew->cmd.exprsInfo, &pCmd->exprsInfo, uid); + + int32_t numOfOutputCols = pNew->cmd.exprsInfo.numOfExprs; + + if (numOfOutputCols > 0) { + int32_t* indexList = calloc(1, numOfOutputCols * sizeof(int32_t)); + for (int32_t i = 0, j = 0; i < pCmd->exprsInfo.numOfExprs; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); + if (pExpr->uid == uid) { + indexList[j++] = i; + } + } + + tscFieldInfoCopy(&pCmd->fieldsInfo, &pNew->cmd.fieldsInfo, indexList, numOfOutputCols); + free(indexList); + + tscFieldInfoUpdateOffset(&pNew->cmd); + } + + pNew->fp = fp; + + pNew->param = param; + pNew->cmd.vnodeIdx = vnodeIndex; + SMeterMetaInfo* pMetermetaInfo = tscGetMeterMetaInfo(pCmd, tableIndex); + + char key[TSDB_MAX_TAGS_LEN + 1] = {0}; + tscGetMetricMetaCacheKey(pCmd, key, pMetermetaInfo->pMeterMeta->uid); + + char* name = pMeterMetaInfo->name; + SMeterMetaInfo* pFinalInfo = NULL; + + if (pPrevSql == NULL) { + SMeterMeta* pMeterMeta = taosGetDataFromCache(tscCacheHandle, name); + SMetricMeta* pMetricMeta = taosGetDataFromCache(tscCacheHandle, key); + + pFinalInfo = tscAddMeterMetaInfo(&pNew->cmd, name, pMeterMeta, pMetricMeta, pMeterMetaInfo->numOfTags, + pMeterMetaInfo->tagColumnIndex); + } else { + SMeterMetaInfo* pPrevInfo = tscGetMeterMetaInfo(&pPrevSql->cmd, 0); + pFinalInfo = tscAddMeterMetaInfo(&pNew->cmd, name, pPrevInfo->pMeterMeta, pPrevInfo->pMetricMeta, pMeterMetaInfo->numOfTags, + pMeterMetaInfo->tagColumnIndex); + + pPrevInfo->pMeterMeta = NULL; + pPrevInfo->pMetricMeta = NULL; + } + + assert(pFinalInfo->pMeterMeta != NULL); + if (UTIL_METER_IS_METRIC(pMetermetaInfo)) { + assert(pFinalInfo->pMetricMeta != NULL); + } + + tscTrace("%p new subquery %p, vnodeIdx:%d, tableIndex:%d, type:%d", pSql, pNew, vnodeIndex, tableIndex, pNew->cmd.type); + return pNew; +} + void tscDoQuery(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; + void* fp = pSql->fp; if (pCmd->command > TSDB_SQL_LOCAL) { tscProcessLocalCmd(pSql); @@ -1153,23 +1802,22 @@ void tscDoQuery(SSqlObj* pSql) { tscAddIntoSqlList(pSql); } - if (tscIsFirstProjQueryOnMetric(pSql)) { - pSql->cmd.vnodeIdx += 1; - } - - void* fp = pSql->fp; - if (pCmd->isInsertFromFile == 1) { tscProcessMultiVnodesInsertForFile(pSql); } else { // pSql may be released in this function if it is a async insertion. tscProcessSql(pSql); - - // handle the multi-vnode insertion for sync model - if (fp == NULL) { - assert(pSql->signature == pSql); - tscProcessMultiVnodesInsert(pSql); - } + if (NULL == fp) tscProcessMultiVnodesInsert(pSql); } } } + +int16_t tscGetJoinTagColIndexByUid(SSqlCmd* pCmd, uint64_t uid) { + STagCond* pTagCond = &pCmd->tagCond; + + if (pTagCond->joinInfo.left.uid == uid) { + return pTagCond->joinInfo.left.tagCol; + } else { + return pTagCond->joinInfo.right.tagCol; + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java index 0e61755b69..7d5b02606e 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java @@ -69,7 +69,7 @@ public enum TSDBError { TSDB_CODE_INVALID_OPTION(45, "invalid option"), TSDB_CODE_NODE_OFFLINE(46, "node offline"), TSDB_CODE_SYNC_REQUIRED(47, "sync required"), - TSDB_CODE_NO_ENOUGH_PNODES(48, "more dnodes are needed"), + TSDB_CODE_NO_ENOUGH_DNODES(48, "more dnodes are needed"), TSDB_CODE_UNSYNCED(49, "node in unsynced state"), TSDB_CODE_TOO_SLOW(50, "too slow"), TSDB_CODE_OTHERS(51, "others"), diff --git a/src/connector/jdbc/src/test/java/TestPreparedStatement.java b/src/connector/jdbc/src/test/java/TestPreparedStatement.java new file mode 100644 index 0000000000..2e2cc0ede6 --- /dev/null +++ b/src/connector/jdbc/src/test/java/TestPreparedStatement.java @@ -0,0 +1,29 @@ +import com.taosdata.jdbc.TSDBDriver; +import com.taosdata.jdbc.TSDBPreparedStatement; + +import java.sql.*; +import java.util.Properties; + +public class TestPreparedStatement { + + public static void main(String[] args) { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "192.168.1.117"); + Connection connection = DriverManager.getConnection("jdbc:TAOS://192.168.1.117:0/?user=root&password=taosdata", properties); + String rawSql = "SELECT ts, c1 FROM (select c1, ts from db.tb1) SUB_QRY"; +// String[] params = new String[]{"ts", "c1"}; + PreparedStatement pstmt = (TSDBPreparedStatement) connection.prepareStatement(rawSql); + ResultSet resSet = pstmt.executeQuery(); + while(resSet.next()) { + for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { + System.out.printf("%d: %s\n", i, resSet.getString(i)); + } + } + + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java b/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java new file mode 100644 index 0000000000..519d7c6ab1 --- /dev/null +++ b/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java @@ -0,0 +1,29 @@ +import com.taosdata.jdbc.TSDBDriver; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.util.Properties; + +public class TestTSDBDatabaseMetaData { + + public static void main(String[] args) { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "192.168.1.114"); + Connection connection = DriverManager.getConnection("jdbc:TAOS://192.168.1.114:0/?user=root&password=taosdata", properties); + DatabaseMetaData dbMetaData = connection.getMetaData(); + ResultSet resSet = dbMetaData.getCatalogs(); + while(resSet.next()) { + for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { + System.out.printf("dbMetaData.getCatalogs(%d) = %s\n", i, resSet.getString(i)); + } + } + + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/src/inc/buildInfo.h b/src/inc/buildInfo.h old mode 100755 new mode 100644 diff --git a/src/inc/lz4.h b/src/inc/lz4.h index d284d63004..43ccb22c9c 100644 --- a/src/inc/lz4.h +++ b/src/inc/lz4.h @@ -75,6 +75,9 @@ extern "C" { * LZ4LIB_API : * Control library symbols visibility. */ + +#include + #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) # define LZ4LIB_API __declspec(dllexport) #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) @@ -392,9 +395,9 @@ typedef struct { * it may change in a future version ! */ #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(uint64_t)) union LZ4_stream_u { - unsigned long long table[LZ4_STREAMSIZE_U64]; + uint64_t table[LZ4_STREAMSIZE_U64]; LZ4_stream_t_internal internal_donotuse; } ; /* previously typedef'd to LZ4_stream_t */ @@ -408,9 +411,9 @@ union LZ4_stream_u { * and may change in a future version ! */ #define LZ4_STREAMDECODESIZE_U64 4 -#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) +#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(uint64_t)) union LZ4_streamDecode_u { - unsigned long long table[LZ4_STREAMDECODESIZE_U64]; + uint64_t table[LZ4_STREAMDECODESIZE_U64]; LZ4_streamDecode_t_internal internal_donotuse; } ; /* previously typedef'd to LZ4_streamDecode_t */ diff --git a/src/inc/sdb.h b/src/inc/sdb.h index 7dd1549f87..4a1969b700 100644 --- a/src/inc/sdb.h +++ b/src/inc/sdb.h @@ -71,6 +71,40 @@ enum _sdbaction { SDB_MAX_ACTION_TYPES }; +#ifdef CLUSTER + +#define SDB_MAX_PEERS 4 +typedef struct { + uint32_t ip; + uint32_t publicIp; + char ipstr[20]; + char zone[12]; + char role; + int64_t createdTime; + uint64_t dbVersion; + int64_t lostTime; + char status; + char numOfMnodes; + int numOfDnodes; + char updateEnd[1]; + + // internal + int syncFd; + void *hbTimer; + void *thandle; + void *pSync; +} SSdbPeer; + +SSdbPeer *sdbAddPeer(uint32_t ip, uint32_t publicIp, char role); + +void sdbUpdateIpList(); + +extern SSdbPeer *sdbPeer[]; +#define sdbInited (sdbPeer[0]) +#define sdbStatus (sdbPeer[0]->status) + +#endif + void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, char keyType, char *directory, void *(*appTool)(char, void *, char *, int, int *)); diff --git a/src/inc/sql.y b/src/inc/sql.y old mode 100755 new mode 100644 index b24902d678..e04324c5de --- a/src/inc/sql.y +++ b/src/inc/sql.y @@ -63,6 +63,7 @@ program ::= cmd. {} cmd ::= SHOW DATABASES. { setDCLSQLElems(pInfo, SHOW_DATABASES, 0);} cmd ::= SHOW MNODES. { setDCLSQLElems(pInfo, SHOW_MNODES, 0);} cmd ::= SHOW DNODES. { setDCLSQLElems(pInfo, SHOW_DNODES, 0);} +cmd ::= SHOW ACCOUNTS. { setDCLSQLElems(pInfo, SHOW_ACCOUNTS, 0);} cmd ::= SHOW USERS. { setDCLSQLElems(pInfo, SHOW_USERS, 0);} cmd ::= SHOW MODULES. { setDCLSQLElems(pInfo, SHOW_MODULES, 0); } @@ -112,7 +113,9 @@ cmd ::= DROP TABLE ifexists(Y) ids(X) cpxName(Z). { } cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDCLSQLElems(pInfo, DROP_DATABASE, 2, &X, &Y); } +cmd ::= DROP DNODE IP(X). { setDCLSQLElems(pInfo, DROP_DNODE, 1, &X); } cmd ::= DROP USER ids(X). { setDCLSQLElems(pInfo, DROP_USER, 1, &X); } +cmd ::= DROP ACCOUNT ids(X). { setDCLSQLElems(pInfo, DROP_ACCOUNT, 1, &X); } /////////////////////////////////THE USE STATEMENT////////////////////////////////////////// cmd ::= USE ids(X). { setDCLSQLElems(pInfo, USE_DATABASE, 1, &X);} @@ -129,8 +132,12 @@ cmd ::= ALTER USER ids(X) PRIVILEGE ids(Y). { setDCLSQLElems(pInfo, ALTER_US cmd ::= ALTER DNODE IP(X) ids(Y). { setDCLSQLElems(pInfo, ALTER_DNODE, 2, &X, &Y); } cmd ::= ALTER DNODE IP(X) ids(Y) ids(Z). { setDCLSQLElems(pInfo, ALTER_DNODE, 3, &X, &Y, &Z); } cmd ::= ALTER LOCAL ids(X). { setDCLSQLElems(pInfo, ALTER_LOCAL, 1, &X); } +cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSQLElems(pInfo, ALTER_LOCAL, 2, &X, &Y); } cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SSQLToken t = {0}; setCreateDBSQL(pInfo, ALTER_DATABASE, &X, &Y, &t);} +cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { SSQLToken t = {0}; setCreateAcctSQL(pInfo, ALTER_ACCT, &X, &t, &Z);} +cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSQL(pInfo, ALTER_ACCT, &X, &Y, &Z);} + // An IDENTIFIER can be a generic identifier, or one of several keywords. // Any non-standard keyword can also be an identifier. // And "ids" is an identifer-or-string. @@ -147,9 +154,53 @@ ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} ifnotexists(X) ::= . {X.n = 0;} /////////////////////////////////THE CREATE STATEMENT/////////////////////////////////////// +//create option for dnode/db/user/account +cmd ::= CREATE DNODE IP(X). { setDCLSQLElems(pInfo, CREATE_DNODE, 1, &X);} +cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). + { setCreateAcctSQL(pInfo, CREATE_ACCOUNT, &X, &Y, &Z);} cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDBSQL(pInfo, CREATE_DATABASE, &X, &Y, &Z);} cmd ::= CREATE USER ids(X) PASS ids(Y). { setDCLSQLElems(pInfo, CREATE_USER, 2, &X, &Y);} +pps(Y) ::= . {Y.n = 0; } +pps(Y) ::= PPS INTEGER(X). {Y = X; } + +tseries(Y) ::= . {Y.n = 0; } +tseries(Y) ::= TSERIES INTEGER(X). {Y = X; } + +dbs(Y) ::= . {Y.n = 0; } +dbs(Y) ::= DBS INTEGER(X). {Y = X; } + +streams(Y) ::= . {Y.n = 0; } +streams(Y) ::= STREAMS INTEGER(X). {Y = X; } + +storage(Y) ::= . {Y.n = 0; } +storage(Y) ::= STORAGE INTEGER(X). {Y = X; } + +qtime(Y) ::= . {Y.n = 0; } +qtime(Y) ::= QTIME INTEGER(X). {Y = X; } + +users(Y) ::= . {Y.n = 0; } +users(Y) ::= USERS INTEGER(X). {Y = X; } + +conns(Y) ::= . {Y.n = 0; } +conns(Y) ::= CONNS INTEGER(X). {Y = X; } + +state(Y) ::= . {Y.n = 0; } +state(Y) ::= STATE ids(X). {Y = X; } + +%type acct_optr {SCreateAcctSQL} +acct_optr(Y) ::= pps(C) tseries(D) storage(P) streams(F) qtime(Q) dbs(E) users(K) conns(L) state(M). { + Y.users = (K.n>0)?atoi(K.z):-1; + Y.dbs = (E.n>0)?atoi(E.z):-1; + Y.tseries = (D.n>0)?atoi(D.z):-1; + Y.streams = (F.n>0)?atoi(F.z):-1; + Y.pps = (C.n>0)?atoi(C.z):-1; + Y.storage = (P.n>0)?strtoll(P.z, NULL, 10):-1; + Y.qtime = (Q.n>0)?strtoll(Q.z, NULL, 10):-1; + Y.conns = (L.n>0)?atoi(L.z):-1; + Y.stat = M; +} + %type keep {tVariantList*} %destructor keep {tVariantListDestroy($$);} keep(Y) ::= KEEP tagitemlist(X). { Y = X; } @@ -183,9 +234,10 @@ db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; } db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; } %type alter_db_optr {SCreateDBInfo} -alter_db_optr(Y) ::= REPLICA tagitem(A). { - Y.replica = A.i64Key; -} +alter_db_optr(Y) ::= . { memset(&Y, 0, sizeof(SCreateDBInfo));} + +alter_db_optr(Y) ::= alter_db_optr(Z) replica(X). { Y = Z; Y.replica = strtol(X.z, NULL, 10); } +alter_db_optr(Y) ::= alter_db_optr(Z) tables(X). { Y = Z; Y.tablesPerVnode = strtol(X.z, NULL, 10); } %type typename {TAOS_FIELD} typename(A) ::= ids(X). { tSQLSetColumnType (&A, &X); } @@ -257,7 +309,7 @@ tagitem(A) ::= INTEGER(X). {toTSDBType(X.type); tVariantCreate(&A, &X); } tagitem(A) ::= FLOAT(X). {toTSDBType(X.type); tVariantCreate(&A, &X); } tagitem(A) ::= STRING(X). {toTSDBType(X.type); tVariantCreate(&A, &X); } tagitem(A) ::= BOOL(X). {toTSDBType(X.type); tVariantCreate(&A, &X); } -tagitem(A) ::= NULL(X). { X.type = TK_STRING; toTSDBType(X.type); tVariantCreate(&A, &X); } +tagitem(A) ::= NULL(X). { X.type = 0; tVariantCreate(&A, &X); } tagitem(A) ::= MINUS(X) INTEGER(Y).{ X.n += Y.n; @@ -273,6 +325,19 @@ tagitem(A) ::= MINUS(X) FLOAT(Y). { tVariantCreate(&A, &X); } +tagitem(A) ::= PLUS(X) INTEGER(Y). { + X.n += Y.n; + X.type = Y.type; + toTSDBType(X.type); + tVariantCreate(&A, &X); +} + +tagitem(A) ::= PLUS(X) FLOAT(Y). { + X.n += Y.n; + X.type = Y.type; + toTSDBType(X.type); + tVariantCreate(&A, &X); +} //////////////////////// The SELECT statement ///////////////////////////////// cmd ::= select(X). { @@ -282,7 +347,7 @@ cmd ::= select(X). { %type select {SQuerySQL*} %destructor select {destroyQuerySql($$);} select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_opt(K) fill_opt(F) sliding_opt(S) groupby_opt(P) orderby_opt(Z) having_opt(N) slimit_opt(G) limit_opt(L). { - A = tSetQuerySQLElems(&T, W, &X, Y, P, Z, &K, &S, F, &L, &G); + A = tSetQuerySQLElems(&T, W, X, Y, P, Z, &K, &S, F, &L, &G); } // selcollist is a list of expressions that are to become the return @@ -313,9 +378,13 @@ as(X) ::= ids(Y). { X = Y; } as(X) ::= . { X.n = 0; } // A complete FROM clause. -%type from {SSQLToken} +%type from {tVariantList*} // current not support query from no-table -from(A) ::= FROM ids(X) cpxName(Y). {A = X; A.n += Y.n;} +from(A) ::= FROM tablelist(X). {A = X;} + +%type tablelist {tVariantList*} +tablelist(A) ::= ids(X) cpxName(Y). { toTSDBType(X.type); X.n += Y.n; A = tVariantListAppendToken(NULL, &X, -1);} +tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z). { toTSDBType(X.type); X.n += Z.n; A = tVariantListAppendToken(Y, &X, -1); } // The value of interval should be the form of "number+[a,s,m,h,d,n,y]" or "now" %type tmvar {SSQLToken} @@ -581,4 +650,4 @@ cmd ::= KILL QUERY IP(X) COLON(Z) INTEGER(Y) COLON(K) INTEGER(F). {X.n += DELIMITERS DESC DETACH EACH END EXPLAIN FAIL FOR GLOB IGNORE IMMEDIATE INITIALLY INSTEAD LIKE MATCH KEY OF OFFSET RAISE REPLACE RESTRICT ROW STATEMENT TRIGGER VIEW ALL COUNT SUM AVG MIN MAX FIRST LAST TOP BOTTOM STDDEV PERCENTILE APERCENTILE LEASTSQUARES HISTOGRAM DIFF - SPREAD WAVG INTERP LAST_ROW NOW IP SEMI NONE PREV LINEAR IMPORT METRIC TBNAME JOIN METRICS STABLE. \ No newline at end of file + SPREAD TWA INTERP LAST_ROW NOW IP SEMI NONE PREV LINEAR IMPORT METRIC TBNAME JOIN METRICS STABLE. \ No newline at end of file diff --git a/src/inc/taos.h b/src/inc/taos.h index 3af3a0ab75..94e99d582c 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -22,11 +22,12 @@ extern "C" { #endif -#define TAOS void -#define TAOS_ROW void ** -#define TAOS_RES void -#define TAOS_SUB void -#define TAOS_STREAM void +typedef void TAOS; +typedef void** TAOS_ROW; +typedef void TAOS_RES; +typedef void TAOS_SUB; +typedef void TAOS_STREAM; +typedef void TAOS_STMT; #define TSDB_DATA_TYPE_NULL 0 #define TSDB_DATA_TYPE_BOOL 1 // 1 bytes @@ -56,11 +57,30 @@ typedef struct taosField { char type; } TAOS_FIELD; -void taos_init(); -int taos_options(TSDB_OPTION option, const void *arg, ...); -TAOS *taos_connect(char *ip, char *user, char *pass, char *db, int port); -void taos_close(TAOS *taos); -int taos_query(TAOS *taos, char *sqlstr); +void taos_init(); +int taos_options(TSDB_OPTION option, const void *arg, ...); +TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, int port); +void taos_close(TAOS *taos); + +typedef struct TAOS_BIND { + int buffer_type; + void * buffer; + unsigned long buffer_length; // unused + unsigned long *length; + int * is_null; + int is_unsigned; // unused + int * error; // unused +} TAOS_BIND; + +TAOS_STMT *taos_stmt_init(TAOS *taos); +int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); +int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind); +int taos_stmt_add_batch(TAOS_STMT *stmt); +int taos_stmt_execute(TAOS_STMT *stmt); +TAOS_RES * taos_stmt_use_result(TAOS_STMT *stmt); +int taos_stmt_close(TAOS_STMT *stmt); + +int taos_query(TAOS *taos, const char *sql); TAOS_RES *taos_use_result(TAOS *taos); TAOS_ROW taos_fetch_row(TAOS_RES *res); int taos_result_precision(TAOS_RES *res); // get the time precision of result @@ -69,35 +89,40 @@ int taos_field_count(TAOS *taos); int taos_num_fields(TAOS_RES *res); int taos_affected_rows(TAOS *taos); TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); -int taos_select_db(TAOS *taos, char *db); +int taos_select_db(TAOS *taos, const char *db); int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); void taos_stop_query(TAOS_RES *res); int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); -int taos_validate_sql(TAOS *taos, char *sql); +int taos_validate_sql(TAOS *taos, const char *sql); // TAOS_RES *taos_list_tables(TAOS *mysql, const char *wild); // TAOS_RES *taos_list_dbs(TAOS *mysql, const char *wild); +// TODO: the return value should be `const` char *taos_get_server_info(TAOS *taos); char *taos_get_client_info(); char *taos_errstr(TAOS *taos); + int taos_errno(TAOS *taos); -void taos_query_a(TAOS *taos, char *sqlstr, void (*fp)(void *param, TAOS_RES *, int code), void *param); +void taos_query_a(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, int code), void *param); void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param); void taos_fetch_row_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), void *param); -TAOS_SUB *taos_subscribe(char *host, char *user, char *pass, char *db, char *table, int64_t time, int mseconds); +TAOS_SUB *taos_subscribe(const char *host, const char *user, const char *pass, const char *db, const char *table, int64_t time, int mseconds); TAOS_ROW taos_consume(TAOS_SUB *tsub); void taos_unsubscribe(TAOS_SUB *tsub); int taos_subfields_count(TAOS_SUB *tsub); TAOS_FIELD *taos_fetch_subfields(TAOS_SUB *tsub); -TAOS_STREAM *taos_open_stream(TAOS *taos, char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), +TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), int64_t stime, void *param, void (*callback)(void *)); void taos_close_stream(TAOS_STREAM *tstr); +int taos_load_table_info(TAOS *taos, const char* tableNameList); + +// TODO: `configDir` should not be declared here extern char configDir[]; // the path to global configuration #ifdef __cplusplus diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 09520f8ed7..1d5e1bb5a5 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -68,7 +68,7 @@ extern "C" { #define TSDB_CODE_INVALID_OPTION 45 #define TSDB_CODE_NODE_OFFLINE 46 #define TSDB_CODE_SYNC_REQUIRED 47 -#define TSDB_CODE_NO_ENOUGH_PNODES 48 +#define TSDB_CODE_NO_ENOUGH_DNODES 48 #define TSDB_CODE_UNSYNCED 49 #define TSDB_CODE_TOO_SLOW 50 #define TSDB_CODE_OTHERS 51 @@ -129,9 +129,11 @@ extern "C" { #define TSDB_CODE_INVALID_QUERY_MSG 106 // failed to validate the sql expression msg by vnode #define TSDB_CODE_CACHE_BLOCK_TS_DISORDERED 107 // time stamp in cache block is disordered #define TSDB_CODE_FILE_BLOCK_TS_DISORDERED 108 // time stamp in file block is disordered -#define TSDB_CODE_INVALID_COMMIT_LOG 109 // invalid commit log may be caused by insufficient sotrage +#define TSDB_CODE_INVALID_COMMIT_LOG 109 // commit log init failed #define TSDB_CODE_SERVER_NO_SPACE 110 -#define TSDB_CODE_INVALID_SUBMIT_MSG 111 +#define TSDB_CODE_NOT_SUPER_TABLE 111 // +#define TSDB_CODE_DUPLICATE_TAGS 112 // tags value for join not unique +#define TSDB_CODE_INVALID_SUBMIT_MSG 113 // message type #define TSDB_MSG_TYPE_REG 1 @@ -196,6 +198,7 @@ extern "C" { #define TSDB_MSG_TYPE_USE_DB_RSP 56 #define TSDB_MSG_TYPE_CREATE_TABLE 57 #define TSDB_MSG_TYPE_CREATE_TABLE_RSP 58 + #define TSDB_MSG_TYPE_DROP_TABLE 59 #define TSDB_MSG_TYPE_DROP_TABLE_RSP 60 #define TSDB_MSG_TYPE_METERINFO 61 @@ -227,6 +230,9 @@ extern "C" { #define TSDB_MSG_TYPE_ALTER_DB 83 #define TSDB_MSG_TYPE_ALTER_DB_RSP 84 +#define TSDB_MSG_TYPE_MULTI_METERINFO 85 +#define TSDB_MSG_TYPE_MULTI_METERINFO_RSP 86 + #define TSDB_MSG_TYPE_HEARTBEAT 91 #define TSDB_MSG_TYPE_HEARTBEAT_RSP 92 #define TSDB_MSG_TYPE_STATUS 93 @@ -236,7 +242,8 @@ extern "C" { #define TSDB_MSG_TYPE_ALTER_ACCT 97 #define TSDB_MSG_TYPE_ALTER_ACCT_RSP 98 -#define TSDB_MSG_TYPE_MAX 99 + +#define TSDB_MSG_TYPE_MAX 101 // IE type #define TSDB_IE_TYPE_SEC 1 @@ -247,17 +254,22 @@ extern "C" { #define TSDB_IE_TYPE_DNODE_EXT 6 #define TSDB_IE_TYPE_DNODE_STATE 7 -// mgmt table enum _mgmt_table { + TSDB_MGMT_TABLE_ACCT, TSDB_MGMT_TABLE_USER, TSDB_MGMT_TABLE_DB, TSDB_MGMT_TABLE_TABLE, TSDB_MGMT_TABLE_PNODE, + TSDB_MGMT_TABLE_MNODE, TSDB_MGMT_TABLE_VGROUP, TSDB_MGMT_TABLE_METRIC, + TSDB_MGMT_TABLE_MODULE, TSDB_MGMT_TABLE_QUERIES, TSDB_MGMT_TABLE_STREAMS, + TSDB_MGMT_TABLE_CONFIGS, TSDB_MGMT_TABLE_CONNS, + TSDB_MGMT_TABLE_SCORES, + TSDB_MGMT_TABLE_GRANTS, TSDB_MGMT_TABLE_MAX, }; @@ -286,15 +298,18 @@ enum _mgmt_table { #define TSDB_METER_STABLE 3 // table created from stream computing #define TSDB_MAX_METER_TYPES 4 -#define TSDB_VN_READ_ACCCESS ((char)0x1) -#define TSDB_VN_WRITE_ACCCESS ((char)0x2) -#define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS|TSDB_VN_WRITE_ACCCESS) +#define TSDB_VN_READ_ACCCESS ((char)0x1) +#define TSDB_VN_WRITE_ACCCESS ((char)0x2) +#define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS) +#define TSDB_COL_NORMAL 0x0U +#define TSDB_COL_TAG 0x1U +#define TSDB_COL_JOIN 0x2U extern char *taosMsg[]; extern char *tsError[]; -#pragma pack(1) +#pragma pack(push, 1) typedef struct { char numOfIps; @@ -410,9 +425,13 @@ typedef struct { short ignoreNotExists; } SDropDbMsg, SUseDbMsg; -typedef struct { char user[TSDB_USER_LEN]; } SDropUserMsg, SDropAcctMsg; +typedef struct { + char user[TSDB_USER_LEN]; +} SDropUserMsg, SDropAcctMsg; -typedef struct { char db[TSDB_DB_NAME_LEN]; } SShowTableMsg; +typedef struct { + char db[TSDB_DB_NAME_LEN]; +} SShowTableMsg; typedef struct { char meterId[TSDB_METER_ID_LEN]; @@ -441,7 +460,9 @@ typedef struct { SSchema schema[]; } SAlterTableMsg; -typedef struct { char db[TSDB_METER_ID_LEN]; } SConnectMsg; +typedef struct { + char db[TSDB_METER_ID_LEN]; +} SConnectMsg; typedef struct { int32_t maxUsers; @@ -470,7 +491,9 @@ typedef struct { char flag; } SCreateUserMsg, SAlterUserMsg; -typedef struct { char db[TSDB_METER_ID_LEN]; } SMgmtHead; +typedef struct { + char db[TSDB_METER_ID_LEN]; +} SMgmtHead; typedef struct { char acctId[TSDB_ACCT_LEN]; @@ -486,13 +509,9 @@ typedef struct { char meterId[TSDB_METER_ID_LEN]; } SRemoveMeterMsg; -typedef struct { short vnode; } SFreeVnodeMsg; - -typedef struct SColIndex { - int16_t colId; - int16_t colIdx; - bool isTag; -} SColIndex; +typedef struct { + short vnode; +} SFreeVnodeMsg; typedef struct SColIndexEx { int16_t colId; @@ -502,12 +521,11 @@ typedef struct SColIndexEx { * whether current meter schema is up-to-date. * * colIdxInBuf is used to denote the index of column in pQuery->colList, - * this value is invalid in client side, as well as in cache block of vnode - * either. + * this value is invalid in client side, as well as in cache block of vnode either. */ - int16_t colIdx; - int16_t colIdxInBuf; - bool isTag; + int16_t colIdx; + int16_t colIdxInBuf; + uint16_t flag; // denote if it is a tag or not } SColIndexEx; /* sql function msg, to describe the message to vnode about sql function @@ -539,23 +557,14 @@ typedef struct SSqlFunctionExpr { SSqlBinaryExprInfo pBinExprInfo; int16_t resBytes; int16_t resType; + int16_t interResBytes; } SSqlFunctionExpr; -typedef struct SColumnFilterMsg { - /* for client side struct, we only need the column id, type, bytes are not - * necessary - * But for data in vnode side, we need all the following information. - * */ - int16_t colId; - int16_t type; - int16_t bytes; - - int16_t filterOn; /* denote if the filter is active */ +typedef struct SColumnFilterInfo { int16_t lowerRelOptr; int16_t upperRelOptr; int16_t filterOnBinary; /* denote if current column is binary */ - /* double/int64_t/float/int share the this memory */ union { struct { int64_t lowerBndi; @@ -570,13 +579,24 @@ typedef struct SColumnFilterMsg { int64_t len; }; }; -} SColumnFilterMsg; +} SColumnFilterInfo; + +/* + * for client side struct, we only need the column id, type, bytes are not necessary + * But for data in vnode side, we need all the following information. + */ +typedef struct SColumnInfo { + int16_t colId; + int16_t type; + int16_t bytes; + int16_t numOfFilters; + SColumnFilterInfo *filters; +} SColumnInfo; /* * enable vnode to understand how to group several tables with different tag; */ typedef struct SMeterSidExtInfo { - // union {int32_t sid; void* pObj;}; int32_t sid; void * pObj; char tags[]; @@ -591,10 +611,8 @@ typedef struct { int16_t vnode; int32_t numOfSids; uint64_t pSidExtInfo; // meter id & tag info ptr, in windows pointer may - // occupy only 4bytes uint64_t uid; - char meterId[TSDB_METER_ID_LEN]; TSKEY skey; TSKEY ekey; int32_t num; @@ -605,13 +623,15 @@ typedef struct { int16_t numOfCols; // the number of columns will be load from vnode char intervalTimeUnit; // time interval type, for revisement of interval(1d) - int64_t nAggTimeInterval; // time interval for aggregation, in million second - uint64_t pTagSchema; // tag schema, used to parse tag information in pSidExtInfo + int64_t nAggTimeInterval; // time interval for aggregation, in million second + + // tag schema, used to parse tag information in pSidExtInfo + uint64_t pTagSchema; int16_t numOfTagsCols; // required number of tags int16_t tagLength; // tag length in current query - int16_t numOfGroupbyCols; // num of group by columns + int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; int16_t orderType; // used in group by xx order by xxx uint64_t groupbyTagIds; @@ -619,7 +639,7 @@ typedef struct { int64_t limit; int64_t offset; - int16_t metricQuery; // denote another query process + int16_t queryType; // denote another query process int16_t numOfOutputCols; // final output columns numbers int16_t interpoType; // interpolate type @@ -628,8 +648,13 @@ typedef struct { int32_t colNameLen; int64_t colNameList; - int64_t pSqlFuncExprs; - SColumnFilterMsg colList[]; + int64_t pSqlFuncExprs; + + int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed + int32_t tsLen; // total length of ts comp block + int32_t tsNumOfBlocks; // ts comp block numbers + int32_t tsOrder; // ts comp block order + SColumnInfo colList[]; } SQueryMeterMsg; typedef struct { @@ -648,12 +673,13 @@ typedef struct { typedef struct { uint64_t qhandle; - char free; + int16_t free; } SRetrieveMeterMsg; typedef struct { int32_t numOfRows; int16_t precision; + int16_t compress; int64_t offset; // updated offset value for multi-vnode projection query int64_t useconds; char data[]; @@ -668,7 +694,8 @@ typedef struct { int64_t totalStorage; int64_t compStorage; int64_t pointsWritten; - char reserved[16]; + uint8_t syncStatus; + uint8_t reserved; } SVnodeLoad; typedef struct { @@ -724,8 +751,9 @@ typedef struct { uint16_t numOfCores; uint8_t alternativeRole; uint8_t reserve; - float memoryAvailable; // MB - float diskAvailable; // GB + uint16_t numOfTotalVnodes; // from config file + uint16_t unused; + float diskAvailable; // GB uint32_t openVnodes; char reserved[16]; SVnodeLoad load[]; @@ -773,22 +801,39 @@ typedef struct { } SMeterInfoMsg; typedef struct { - char meterId[TSDB_METER_ID_LEN]; + int32_t numOfMeters; + char meterId[]; +} SMultiMeterInfoMsg; - int16_t numOfGroupbyCols; // num of group by columns - int16_t orderIndex; - int16_t orderType; // used in group by xx order by xxx - uint64_t groupbyTagIds; +typedef struct { + int16_t elemLen; + + char meterId[TSDB_METER_ID_LEN]; + int16_t orderIndex; + int16_t orderType; // used in group by xx order by xxx + + int16_t rel; // denotes the relation between condition and table list + + int32_t tableCond; // offset value of table name condition + int32_t tableCondLen; + + int32_t cond; // offset of column query condition + int32_t condLen; int16_t tagCols[TSDB_MAX_TAGS + 1]; // required tag columns, plus one is for table name int16_t numOfTags; // required number of tags - int64_t limit; - int64_t offset; + int16_t numOfGroupCols; // num of group by columns + int32_t groupbyTagColumnList; +} SMetricMetaElemMsg; + +typedef struct { + int32_t numOfMeters; + int32_t join; + int32_t joinCondLen; // for join condition + + int32_t metaElem[TSDB_MAX_JOIN_TABLE_NUM]; - int32_t condLength; - int16_t type; // denotes if it has the meter id pools - char tags[]; } SMetricMetaMsg; typedef struct { @@ -806,24 +851,28 @@ typedef struct { } SMetricMeta; typedef struct SMeterMeta { - int16_t numOfTags; - int16_t precision; + uint8_t numOfTags : 6; + uint8_t precision : 2; + uint8_t meterType : 4; + uint8_t index : 4; // used locally + int16_t numOfColumns; - int32_t sversion; - uint64_t pSchema; - SVPeerDesc vpeerDesc[TSDB_VNODES_SUPPORT]; + int16_t rowSize; // used locally, calculated in client + int16_t sversion; - int32_t sid; - int32_t vgid; + SVPeerDesc vpeerDesc[TSDB_VNODES_SUPPORT]; + int32_t sid; + int32_t vgid; uint64_t uid; - int16_t meterType; - int16_t index; // used locally - int32_t rowSize; // used locally, calculated in client - uint64_t tags; } SMeterMeta; +typedef struct SMultiMeterMeta { + char meterId[TSDB_METER_ID_LEN]; // note: This field must be at the front + SMeterMeta meta; +} SMultiMeterMeta; + typedef struct { char name[TSDB_METER_ID_LEN]; char data[TSDB_MAX_TAGS_LEN]; @@ -840,7 +889,9 @@ typedef struct { char payload[]; } SShowMsg; -typedef struct { char ip[20]; } SCreateMnodeMsg, SDropMnodeMsg, SCreateDnodeMsg, SDropDnodeMsg; +typedef struct { + char ip[20]; +} SCreateMnodeMsg, SDropMnodeMsg, SCreateDnodeMsg, SDropDnodeMsg; typedef struct { uint64_t qhandle; @@ -852,7 +903,9 @@ typedef struct { int32_t sid; } SMeterCfgMsg; -typedef struct { int32_t vnode; } SVpeerCfgMsg; +typedef struct { + int32_t vnode; +} SVpeerCfgMsg; typedef struct { char ip[20]; @@ -907,7 +960,7 @@ typedef struct { char status; } SAlterStreamMsg; -#pragma pack() +#pragma pack(pop) #ifdef __cplusplus } diff --git a/src/inc/tast.h b/src/inc/tast.h index 37b7765ee2..798a7f89a0 100644 --- a/src/inc/tast.h +++ b/src/inc/tast.h @@ -24,6 +24,7 @@ extern "C" { #include #include +#include "taosmsg.h" #include "tsql.h" struct tSQLBinaryExpr; @@ -37,6 +38,28 @@ enum { TSQL_NODE_VALUE = 0x4, }; +typedef bool (*__result_filter_fn_t)(const void *, void *); +typedef void (*__do_filter_suppl_fn_t)(void *, void *); + +/** + * this structure is used to filter data in tags, so the offset of filtered tag column in tagdata string is required + * + */ +typedef struct tQueryInfo { + int32_t offset; // offset value in tags + int32_t colIdx; // index of column in schema + uint8_t optr; // expression operator + SSchema sch; // schema of tags + tVariant q; // query condition value on the specific schema, filter expression + __compar_fn_t compare; // filter function +} tQueryInfo; + +typedef struct SBinaryFilterSupp { + __result_filter_fn_t fp; + __do_filter_suppl_fn_t setupInfoFn; + void * pExtInfo; +} SBinaryFilterSupp; + typedef struct tSQLSyntaxNode { uint8_t nodeType; int16_t colId; // for schema, the id of column @@ -48,48 +71,37 @@ typedef struct tSQLSyntaxNode { } tSQLSyntaxNode; typedef struct tSQLBinaryExpr { - uint8_t nSQLBinaryOptr; - uint8_t filterOnPrimaryKey; // 0: do not contain primary filter, 1: contain - // primary key - - tSQLSyntaxNode *pLeft; - tSQLSyntaxNode *pRight; + uint8_t nSQLBinaryOptr; // filter operator + uint8_t filterOnPrimaryKey; // 0: do not contain primary filter, 1: contain + + /* + * provide the information to support filter operation on this expression + * only available for leaf node + */ + void * info; + tSQLSyntaxNode *pLeft; // left child pointer + tSQLSyntaxNode *pRight; // right child pointer } tSQLBinaryExpr; -#define TAST_NODE_TYPE_INDEX_ENTRY 0 -#define TAST_NODE_TYPE_METER_PTR 1 - typedef struct tQueryResultset { void ** pRes; int64_t num; - int32_t nodeType; } tQueryResultset; -typedef struct tQueryInfo { - int32_t offset; // offset value in tags - int32_t colIdx; // index of column in schema - struct SSchema *pSchema; // schema of tags - tVariant q; // queries cond - uint8_t optr; - __compar_fn_t comparator; -} tQueryInfo; - -void tSQLBinaryExprFromString(tSQLBinaryExpr **pExpr, struct SSchema *pSchema, int32_t numOfCols, char *src, - int32_t len); +void tSQLBinaryExprFromString(tSQLBinaryExpr **pExpr, SSchema *pSchema, int32_t numOfCols, char *src, int32_t len); void tSQLBinaryExprToString(tSQLBinaryExpr *pExpr, char *dst, int32_t *len); -void tSQLBinaryExprDestroy(tSQLBinaryExpr **pExprs); +void tSQLBinaryExprDestroy(tSQLBinaryExpr **pExprs, void (*fp)(void*)); -void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExprs, struct tSkipList *pSkipList, struct SSchema *pSchema, - int32_t numOfCols, bool (*fp)(struct tSkipListNode *, void *), tQueryResultset *result); +void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExprs, struct tSkipList *pSkipList, tQueryResultset *result, + SBinaryFilterSupp *param); void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, char *, int32_t)); void tSQLBinaryExprTrv(tSQLBinaryExpr *pExprs, int32_t *val, int16_t *ids); - -bool tSQLElemFilterCallback(struct tSkipListNode *pNode, void *param); +void tQueryResultClean(tQueryResultset *pRes); #ifdef __cplusplus } diff --git a/src/inc/textbuffer.h b/src/inc/textbuffer.h index e8f829bb55..c7de20bd74 100644 --- a/src/inc/textbuffer.h +++ b/src/inc/textbuffer.h @@ -123,7 +123,7 @@ typedef struct tExtMemBuffer { EXT_BUFFER_FLUSH_MODEL flushModel; } tExtMemBuffer; -void getExtTmpfilePath(const char *fileNamePattern, int64_t serialNumber, int32_t seg, int32_t slot, char *dstPath); +void getTmpfilePath(const char *fileNamePattern, char *dstPath); /* * create ext-memory buffer @@ -253,10 +253,10 @@ int32_t compare_a(tOrderDescriptor *, int32_t numOfRow1, int32_t s1, char *data1 int32_t compare_d(tOrderDescriptor *, int32_t numOfRow1, int32_t s1, char *data1, int32_t numOfRow2, int32_t s2, char *data2); -void tMemBucketCreate(tMemBucket **pBucket, int32_t totalSlots, int32_t nBufferSize, int16_t nElemSize, +tMemBucket* tMemBucketCreate(int32_t totalSlots, int32_t nBufferSize, int16_t nElemSize, int16_t dataType, tOrderDescriptor *pDesc); -void tMemBucketDestroy(tMemBucket **pBucket); +void tMemBucketDestroy(tMemBucket *pBucket); void tMemBucketPut(tMemBucket *pBucket, void *data, int32_t numOfRows); diff --git a/src/inc/tglobalcfg.h b/src/inc/tglobalcfg.h index 95a3cc57e2..e83c89593f 100644 --- a/src/inc/tglobalcfg.h +++ b/src/inc/tglobalcfg.h @@ -24,11 +24,15 @@ extern "C" { #include #include "tsdb.h" +extern int (*startMonitor)(); +extern void (*stopMonitor)(); + // system info extern int64_t tsPageSize; extern int64_t tsOpenMax; extern int64_t tsStreamMax; extern int32_t tsNumOfCores; +extern int32_t tsAlternativeRole; extern float tsTotalLogDirGB; extern float tsTotalTmpDirGB; extern float tsTotalDataDirGB; @@ -58,7 +62,6 @@ extern short tsMgmtShellPort; extern short tsVnodeShellPort; extern short tsVnodeVnodePort; extern short tsMgmtMgmtPort; -extern short tsVnodeSyncPort; extern short tsMgmtSyncPort; extern int tsStatusInterval; @@ -70,12 +73,14 @@ extern int tsMetricMetaKeepTimer; extern float tsNumOfThreadsPerCore; extern float tsRatioOfQueryThreads; +extern char tsPublicIp[]; extern char tsInternalIp[]; +extern char tsPrivateIp[]; extern char tsServerIpStr[]; extern short tsNumOfVnodesPerCore; extern short tsNumOfTotalVnodes; -extern int tsShellsPerVnode; extern short tsCheckHeaderFile; +extern uint32_t tsServerIp; extern int tsSessionsPerVnode; extern int tsAverageCacheBlocks; @@ -121,6 +126,9 @@ extern int tsMgmtEqualVnodeNum; extern int tsEnableHttpModule; extern int tsEnableMonitorModule; extern int tsRestRowLimit; +extern int tsCompressMsgSize; + +extern char tsSocketType[4]; extern int tsTimePrecision; extern int tsMinSlidingTime; @@ -132,14 +140,13 @@ extern int tsStreamCompRetryDelay; extern int tsProjectExecInterval; extern int64_t tsMaxRetentWindow; -extern char tsSocketType[4]; - extern char tsHttpIp[]; extern short tsHttpPort; extern int tsHttpCacheSessions; extern int tsHttpSessionExpire; extern int tsHttpMaxThreads; extern int tsHttpEnableCompress; +extern int tsHttpEnableRecordSql; extern int tsTelegrafUseFieldNum; extern int tsAdminRowLimit; @@ -177,25 +184,30 @@ extern char tsCharset[64]; // default encode string // void tsReadGlobalLogConfig(); bool tsReadGlobalConfig(); +bool tsReadGlobalConfigSpec(); int tsCfgDynamicOptions(char *msg); void tsPrintGlobalConfig(); +void tsPrintGlobalConfigSpec(); void tsSetAllDebugFlag(); void tsSetTimeZone(); void tsSetLocale(); void tsInitGlobalConfig(); - -#define TSDB_CFG_CTYPE_B_CONFIG 1U // can be configured from file -#define TSDB_CFG_CTYPE_B_SHOW 2U // can displayed by "show configs" commands -#define TSDB_CFG_CTYPE_B_LOG 4U // is a log type configuration -#define TSDB_CFG_CTYPE_B_CLIENT 8U // can be displayed in the client log -#define TSDB_CFG_CTYPE_B_OPTION 16U // can be configured by taos_options function -#define TSDB_CFG_CTYPE_B_NOT_PRINT 32U - -#define TSDB_CFG_CSTATUS_NONE 0 // not configured -#define TSDB_CFG_CSTATUS_DEFAULT 1 // use system default value -#define TSDB_CFG_CSTATUS_FILE 2 // configured from file -#define TSDB_CFG_CSTATUS_OPTION 3 // configured by taos_options function -#define TSDB_CFG_CSTATUS_ARG 4 // configured by program argument +void tsExpandFilePath(char* option_name, char* input_value); + +#define TSDB_CFG_CTYPE_B_CONFIG 1U // can be configured from file +#define TSDB_CFG_CTYPE_B_SHOW 2U // can displayed by "show configs" commands +#define TSDB_CFG_CTYPE_B_LOG 4U // is a log type configuration +#define TSDB_CFG_CTYPE_B_CLIENT 8U // can be displayed in the client log +#define TSDB_CFG_CTYPE_B_OPTION 16U // can be configured by taos_options function +#define TSDB_CFG_CTYPE_B_NOT_PRINT 32U // such as password +#define TSDB_CFG_CTYPE_B_LITE 64U // is a lite type configuration +#define TSDB_CFG_CTYPE_B_CLUSTER 128U // is a cluster type configuration + +#define TSDB_CFG_CSTATUS_NONE 0 // not configured +#define TSDB_CFG_CSTATUS_DEFAULT 1 // use system default value +#define TSDB_CFG_CSTATUS_FILE 2 // configured from file +#define TSDB_CFG_CSTATUS_OPTION 3 // configured by taos_options function +#define TSDB_CFG_CSTATUS_ARG 4 // configured by program argument enum { TSDB_CFG_VTYPE_SHORT, diff --git a/src/inc/tidpool.h b/src/inc/tidpool.h index 2b1ac2f2e6..08f9d36467 100644 --- a/src/inc/tidpool.h +++ b/src/inc/tidpool.h @@ -22,6 +22,10 @@ extern "C" { void *taosInitIdPool(int maxId); +int taosUpdateIdPool(void *handle, int maxId); + +int taosIdPoolMaxSize(void *handle); + int taosAllocateId(void *handle); void taosFreeId(void *handle, int id); diff --git a/src/inc/trpc.h b/src/inc/trpc.h index f849d15d06..3ae6e9ea97 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -21,6 +21,7 @@ extern "C" { #include "taosmsg.h" #include "tsched.h" +#include "tglobalcfg.h" #define TAOS_CONN_UDPS 0 #define TAOS_CONN_UDPC 1 @@ -41,7 +42,7 @@ extern "C" { #define TAOS_CONN_SOCKET_TYPE_C() ((strcasecmp(tsSocketType, TAOS_SOCKET_TYPE_NAME_UDP) == 0)? TAOS_CONN_UDP:TAOS_CONN_TCPC) #define taosSendMsgToPeer(x, y, z) taosSendMsgToPeerH(x, y, z, NULL) -#define taosOpenRpcChann(x, y, z) taosOpenRpcChannWithQ(x, y, z, NULL) +#define taosOpenRpcChann(x, y, z) taosOpenRpcChannWithQ(x,y,z,NULL) #define taosBuildReqMsg(x, y) taosBuildReqMsgWithSize(x, y, 512) #define taosBuildRspMsg(x, y) taosBuildRspMsgWithSize(x, y, 512) diff --git a/src/inc/tschemautil.h b/src/inc/tschemautil.h index 96a3cb8047..0b8a2d6a93 100644 --- a/src/inc/tschemautil.h +++ b/src/inc/tschemautil.h @@ -22,6 +22,7 @@ extern "C" { #include #include "taosmsg.h" +#include "tstoken.h" #define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) @@ -45,7 +46,7 @@ struct SSchema *tsGetSchema(SMeterMeta *pMeta); struct SSchema *tsGetTagSchema(SMeterMeta *pMeta); -struct SSchema *tsGetSchemaColIdx(SMeterMeta *pMeta, int32_t startCol); +struct SSchema *tsGetColumnSchema(SMeterMeta *pMeta, int32_t startCol); char *tsGetTagsValue(SMeterMeta *pMeta); @@ -53,7 +54,9 @@ bool tsMeterMetaIdentical(SMeterMeta *p1, SMeterMeta *p2); void extractMeterName(char *meterId, char *name); -void extractDBName(char *meterId, char *name); +SSQLToken extractDBName(char *meterId, char *name); + +void extractTableNameFromToken(SSQLToken *pToken, SSQLToken* pTable); #ifdef __cplusplus } diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 17070c617f..de4e430a46 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -31,10 +31,10 @@ extern "C" { #define TSKEY int64_t #endif -#define TSDB_TRUE 1 -#define TSDB_FALSE 0 -#define TSDB_OK 0 -#define TSDB_ERR -1 +#define TSDB_TRUE 1 +#define TSDB_FALSE 0 +#define TSDB_OK 0 +#define TSDB_ERR -1 #define TS_PATH_DELIMITER "." @@ -60,23 +60,25 @@ enum _syncstatus { TSDB_SSTATUS_SYNC_FILE, }; -#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes -#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte -#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes -#define TSDB_DATA_TYPE_INT 4 // 4 bytes -#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes -#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes -#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes -#define TSDB_DATA_TYPE_BINARY 8 // string -#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes -#define TSDB_DATA_TYPE_NCHAR 10 // wide string +#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes +#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte +#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes +#define TSDB_DATA_TYPE_INT 4 // 4 bytes +#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes +#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes +#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes +#define TSDB_DATA_TYPE_BINARY 8 // string +#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes +#define TSDB_DATA_TYPE_NCHAR 10 // unicode string #define TSDB_KEYSIZE sizeof(TSKEY) + #if LINUX #define TSDB_NCHAR_SIZE sizeof(wchar_t) #else #define TSDB_NCHAR_SIZE 4 #endif +//#define TSDB_CHAR_TERMINATED_SPACE 1 #define TSDB_RELATION_INVALID 0 #define TSDB_RELATION_LESS 1 @@ -96,31 +98,28 @@ enum _syncstatus { #define TSDB_BINARY_OP_MULTIPLY 13 #define TSDB_BINARY_OP_DIVIDE 14 #define TSDB_BINARY_OP_REMAINDER 15 - #define TSDB_USERID_LEN 9 #define TS_PATH_DELIMITER_LEN 1 -#define TSDB_METER_ID_LEN_MARGIN 10 -#define TSDB_METER_ID_LEN \ - (TSDB_DB_NAME_LEN + TSDB_METER_NAME_LEN + 2 * TS_PATH_DELIMITER_LEN + TSDB_USERID_LEN + \ - TSDB_METER_ID_LEN_MARGIN) // TSDB_DB_NAME_LEN+TSDB_METER_NAME_LEN+2*strlen(TS_PATH_DELIMITER)+strlen(USERID) +#define TSDB_METER_ID_LEN_MARGIN 10 +#define TSDB_METER_ID_LEN (TSDB_DB_NAME_LEN+TSDB_METER_NAME_LEN+2*TS_PATH_DELIMITER_LEN+TSDB_USERID_LEN+TSDB_METER_ID_LEN_MARGIN) //TSDB_DB_NAME_LEN+TSDB_METER_NAME_LEN+2*strlen(TS_PATH_DELIMITER)+strlen(USERID) #define TSDB_UNI_LEN 24 #define TSDB_USER_LEN TSDB_UNI_LEN #define TSDB_ACCT_LEN TSDB_UNI_LEN #define TSDB_PASSWORD_LEN TSDB_UNI_LEN #define TSDB_MAX_COLUMNS 256 -#define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns +#define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns #define TSDB_METER_NAME_LEN 64 #define TSDB_DB_NAME_LEN 32 #define TSDB_COL_NAME_LEN 64 -#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 16 -#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE +#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 16 +#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE -#define TSDB_MAX_BYTES_PER_ROW TSDB_MAX_COLUMNS * 16 +#define TSDB_MAX_BYTES_PER_ROW TSDB_MAX_COLUMNS * 16 #define TSDB_MAX_TAGS_LEN 512 -#define TSDB_MAX_TAGS 6 +#define TSDB_MAX_TAGS 32 #define TSDB_AUTH_LEN 16 #define TSDB_KEY_LEN 16 @@ -134,7 +133,7 @@ enum _syncstatus { #define TSDB_LOCALE_LEN 64 #define TSDB_TIMEZONE_LEN 64 -#define TSDB_IPv4ADDR_LEN 16 +#define TSDB_IPv4ADDR_LEN 16 #define TSDB_FILENAME_LEN 128 #define TSDB_METER_VNODE_BITS 20 #define TSDB_METER_SID_MASK 0xFFFFF @@ -146,11 +145,11 @@ enum _syncstatus { #define TSDB_METER_STATE_OFFLINE 0 #define TSDB_METER_STATE_ONLLINE 1 -#define TSDB_DEFAULT_PKT_SIZE 65480 // same as RPC_MAX_UDP_SIZE +#define TSDB_DEFAULT_PKT_SIZE 65480 //same as RPC_MAX_UDP_SIZE -#define TSDB_PAYLOAD_SIZE (TSDB_DEFAULT_PKT_SIZE - 100) -#define TSDB_DEFAULT_PAYLOAD_SIZE 1024 // default payload size -#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth +#define TSDB_PAYLOAD_SIZE (TSDB_DEFAULT_PKT_SIZE - 100) +#define TSDB_DEFAULT_PAYLOAD_SIZE 1024 // default payload size +#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth #define TSDB_SQLCMD_SIZE 1024 #define TSDB_MAX_VNODES 256 #define TSDB_MIN_VNODES 50 @@ -161,60 +160,81 @@ enum _syncstatus { #define TSDB_DNODE_ROLE_VNODE 2 #define TSDB_MAX_MPEERS 5 -#define TSDB_MAX_MGMT_IPS (TSDB_MAX_MPEERS + 1) +#define TSDB_MAX_MGMT_IPS (TSDB_MAX_MPEERS+1) + +//#define TSDB_REPLICA_MAX_NUM 3 +#define TSDB_REPLICA_MIN_NUM 1 + +#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta -#define TSDB_REPLICA_MAX_NUM 3 -#define TSDB_REPLICA_MIN_NUM 1 +//default value == 10 +#define TSDB_FILE_MIN_PARTITION_RANGE 1 //minimum partition range of vnode file in days +#define TSDB_FILE_MAX_PARTITION_RANGE 3650 //max partition range of vnode file in days -// default value == 10 -#define TSDB_FILE_MIN_PARTITION_RANGE 1 // minimum partition range of vnode file in days -#define TSDB_FILE_MAX_PARTITION_RANGE 3650 // max partition range of vnode file in days +#define TSDB_DATA_MIN_RESERVE_DAY 1 // data in db to be reserved. +#define TSDB_DATA_DEFAULT_RESERVE_DAY 3650 // ten years -#define TSDB_DATA_MIN_RESERVE_DAY 1 // data in db to be reserved. -#define TSDB_DATA_DEFAULT_RESERVE_DAY 3650 // ten years +#define TSDB_MIN_COMPRESSION_LEVEL 0 +#define TSDB_MAX_COMPRESSION_LEVEL 2 -#define TSDB_MIN_COMPRESSION_LEVEL 0 -#define TSDB_MAX_COMPRESSION_LEVEL 2 +#define TSDB_MIN_CACHE_BLOCKS_PER_METER 32 +#define TSDB_MAX_CACHE_BLOCKS_PER_METER 40960 -#define TSDB_MIN_CACHE_BLOCKS_PER_METER 32 -#define TSDB_MAX_CACHE_BLOCKS_PER_METER 40960 +#define TSDB_MIN_COMMIT_TIME_INTERVAL 30 +#define TSDB_MAX_COMMIT_TIME_INTERVAL 40960 -#define TSDB_MIN_COMMIT_TIME_INTERVAL 30 -#define TSDB_MAX_COMMIT_TIME_INTERVAL 40960 +#define TSDB_MIN_ROWS_IN_FILEBLOCK 200 +#define TSDB_MAX_ROWS_IN_FILEBLOCK 500000 -#define TSDB_MIN_ROWS_IN_FILEBLOCK 200 -#define TSDB_MAX_ROWS_IN_FILEBLOCK 500000 +#define TSDB_MIN_CACHE_BLOCK_SIZE 100 +#define TSDB_MAX_CACHE_BLOCK_SIZE 104857600 -#define TSDB_MIN_CACHE_BLOCK_SIZE 100 -#define TSDB_MAX_CACHE_BLOCK_SIZE 104857600 +#define TSDB_MIN_CACHE_BLOCKS 100 +#define TSDB_MAX_CACHE_BLOCKS 409600 -#define TSDB_MIN_CACHE_BLOCKS 100 -#define TSDB_MAX_CACHE_BLOCKS 409600 +#define TSDB_MAX_AVG_BLOCKS 2048 -#define TSDB_MAX_AVG_BLOCKS 2048 +#define TSDB_MIN_TABLES_PER_VNODE 1 +#define TSDB_MAX_TABLES_PER_VNODE 220000 -#define TSDB_MIN_TABLES_PER_VNODE 1 -#define TSDB_MAX_TABLES_PER_VNODE 220000 +#define TSDB_MAX_JOIN_TABLE_NUM 5 -#define TSDB_MAX_BINARY_LEN (TSDB_MAX_BYTES_PER_ROW - TSDB_KEYSIZE) -#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_BYTES_PER_ROW - TSDB_KEYSIZE) -#define PRIMARYKEY_TIMESTAMP_COL_INDEX 0 +#define TSDB_MAX_BINARY_LEN (TSDB_MAX_BYTES_PER_ROW-TSDB_KEYSIZE) +#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_BYTES_PER_ROW-TSDB_KEYSIZE) +#define PRIMARYKEY_TIMESTAMP_COL_INDEX 0 -#define TSDB_DATA_BOOL_NULL 0x02 -#define TSDB_DATA_TINYINT_NULL 0x80 -#define TSDB_DATA_SMALLINT_NULL 0x8000 -#define TSDB_DATA_INT_NULL 0x80000000 -#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L +#define TSDB_DATA_BOOL_NULL 0x02 +#define TSDB_DATA_TINYINT_NULL 0x80 +#define TSDB_DATA_SMALLINT_NULL 0x8000 +#define TSDB_DATA_INT_NULL 0x80000000 +#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L -#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN -#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN -#define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF -#define TSDB_DATA_BINARY_NULL 0xFF +#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN +#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN +#define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF +#define TSDB_DATA_BINARY_NULL 0xFF -#define TSDB_DATA_NULL_STR "NULL" -#define TSDB_DATA_NULL_STR_L "null" +#define TSDB_DATA_NULL_STR "NULL" +#define TSDB_DATA_NULL_STR_L "null" -#define TSDB_MAX_RPC_THREADS 5 +#define TSDB_MAX_RPC_THREADS 5 + +#define TSDB_QUERY_TYPE_QUERY 0 // normal query +#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x1 // free qhandle at vnode + +/* + * 1. ordinary sub query for select * from super_table + * 2. all sqlobj generated by createSubqueryObj with this flag + */ +#define TSDB_QUERY_TYPE_SUBQUERY 0x2 +#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x4 // two-stage subquery for super table + +#define TSDB_QUERY_TYPE_TABLE_QUERY 0x8 // query ordinary table; below only apply to client side +#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10 // query on super table +#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20 // join query +#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40 // select *,columns... query +#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80 // join sub query at the second stage #ifdef __cplusplus } diff --git a/src/inc/tskiplist.h b/src/inc/tskiplist.h index 3ebbd5a610..d6cdd77c11 100644 --- a/src/inc/tskiplist.h +++ b/src/inc/tskiplist.h @@ -43,8 +43,8 @@ typedef enum tSkipListPointQueryType { } tSkipListPointQueryType; typedef struct tSkipListNode { - uint16_t nLevel; - char * pData; + uint16_t nLevel; + char * pData; struct tSkipListNode **pForward; struct tSkipListNode **pBackward; @@ -55,32 +55,22 @@ typedef struct tSkipListNode { /* * @version 0.2 * @date 2017/11/12 - * @author liaohj * the simple version of SkipList. - * for multi-thread safe purpose, we employ pthread_rwlock_t to guarantee to - * generate + * for multi-thread safe purpose, we employ pthread_rwlock_t to guarantee to generate * deterministic result. Later, we will remove the lock in SkipList to further - * enhance the performance. In this case, one should use the concurrent skip - * list (by - * using michael-scott algorithm) instead of this simple version in a - * multi-thread + * enhance the performance. In this case, one should use the concurrent skip list (by + * using michael-scott algorithm) instead of this simple version in a multi-thread * environment, to achieve higher performance of read/write operations. * * Note: Duplicated primary key situation. - * In case of duplicated primary key, two ways can be employed to handle this - * situation: + * In case of duplicated primary key, two ways can be employed to handle this situation: * 1. add as normal insertion with out special process. - * 2. add an overflow pointer at each list node, all nodes with the same key - * will be added - * in the overflow pointer. In this case, the total steps of each search will - * be reduced significantly. - * Currently, we implement the skip list in a line with the first means, - * maybe refactor it soon. - * Memory consumption: the memory alignment causes many memory wasted. So, - * employ a memory - * pool will significantly reduce the total memory consumption, as well as the - * calloc/malloc - * operation costs. + * 2. add an overflow pointer at each list node, all nodes with the same key will be added + * in the overflow pointer. In this case, the total steps of each search will be reduced significantly. + * Currently, we implement the skip list in a line with the first means, maybe refactor it soon. + * + * Memory consumption: the memory alignment causes many memory wasted. So, employ a memory + * pool will significantly reduce the total memory consumption, as well as the calloc/malloc operation costs. * * 3. use the iterator pattern to refactor all routines to make it more clean */ @@ -94,7 +84,6 @@ typedef struct tSkipListState { // in bytes, sizeof(tSkipList)+sizeof(tSkipListNode)*tSkipList->nSize uint64_t nTotalMemSize; uint64_t nLevelNodeCnt[MAX_SKIP_LIST_LEVEL]; - uint64_t queryCount; // total query count /* @@ -115,38 +104,43 @@ typedef struct tSkipListState { typedef struct tSkipList { tSkipListNode pHead; uint64_t nSize; - - uint16_t nMaxLevel; - uint16_t nLevel; - - uint16_t keyType; - uint16_t nMaxKeyLen; + uint16_t nMaxLevel; + uint16_t nLevel; + uint16_t keyType; + uint16_t nMaxKeyLen; __compar_fn_t comparator; - pthread_rwlock_t lock; // will be removed soon - - // skiplist state - tSkipListState state; + pthread_rwlock_t lock; // will be removed soon + tSkipListState state; // skiplist state } tSkipList; +/* + * iterate the skiplist + * this will cause the multi-thread problem, when the skiplist is destroyed, the iterate may + * continue iterating the skiplist, so add the reference count for skiplist + * TODO add the ref for skiplist when one iterator is created + */ +typedef struct SSkipListIterator { + tSkipList * pSkipList; + tSkipListNode *cur; + int64_t num; +} SSkipListIterator; + /* * query condition structure to denote the range query - * //todo merge the point query cond with range query condition + * todo merge the point query cond with range query condition */ typedef struct tSKipListQueryCond { // when the upper bounding == lower bounding, it is a point query tSkipListKey lowerBnd; tSkipListKey upperBnd; - - int32_t lowerBndRelOptr; // relation operator to denote if lower bound is - - // included or not - int32_t upperBndRelOptr; + int32_t lowerBndRelOptr; // relation operator to denote if lower bound is + int32_t upperBndRelOptr; // included or not } tSKipListQueryCond; -tSkipList* tSkipListCreate(int16_t nMaxLevel, int16_t keyType, int16_t nMaxKeyLen); +tSkipList *tSkipListCreate(int16_t nMaxLevel, int16_t keyType, int16_t nMaxKeyLen); -void* tSkipListDestroy(tSkipList *pSkipList); +void *tSkipListDestroy(tSkipList *pSkipList); // create skip list key tSkipListKey tSkipListCreateKey(int32_t type, char *val, size_t keyLength); @@ -200,7 +194,10 @@ int32_t tSkipListQuery(tSkipList *pSkipList, tSKipListQueryCond *pQueryCond, tSk int32_t tSkipListPointQuery(tSkipList *pSkipList, tSkipListKey *pKey, int32_t numOfKey, tSkipListPointQueryType type, tSkipListNode ***pResult); -void removeNodeEachLevel(tSkipList *pSkipList, int32_t nLevel); +int32_t tSkipListIteratorReset(tSkipList *pSkipList, SSkipListIterator *iter); +bool tSkipListIteratorNext(SSkipListIterator *iter); +tSkipListNode *tSkipListIteratorGet(SSkipListIterator *iter); + #ifdef __cplusplus } #endif diff --git a/src/inc/tsql.h b/src/inc/tsql.h index d0cf049a24..0bcd8ffa68 100644 --- a/src/inc/tsql.h +++ b/src/inc/tsql.h @@ -24,28 +24,27 @@ extern "C" { #include "tsqldef.h" #include "ttypes.h" -#define TK_SPACE 200 -#define TK_COMMENT 201 -#define TK_ILLEGAL 202 -#define TK_HEX 203 -#define TK_OCT 204 +#define TK_SPACE 200 +#define TK_COMMENT 201 +#define TK_ILLEGAL 202 +#define TK_HEX 203 // hex number 0x123 +#define TK_OCT 204 // oct number +#define TK_BIN 205 // bin format data 0b111 +#define TK_FILE 206 -#define TSQL_SO_ASC 1 -#define TSQL_SO_DESC 0 +#define TSQL_SO_ASC 1 +#define TSQL_SO_DESC 0 #define MAX_TOKEN_LEN 30 -#define TSQL_TBNAME "TBNAME" +#define TSQL_TBNAME "TBNAME" #define TSQL_TBNAME_L "tbname" -#define TSQL_STABLE_QTYPE_COND 1 -#define TSQL_STABLE_QTYPE_SET 2 - // token type enum { - TSQL_NODE_TYPE_EXPR = 0x1, - TSQL_NODE_TYPE_ID = 0x2, - TSQL_NODE_TYPE_VALUE = 0x4, + TSQL_NODE_TYPE_EXPR = 0x1, + TSQL_NODE_TYPE_ID = 0x2, + TSQL_NODE_TYPE_VALUE = 0x4, }; extern char tTokenTypeSwitcher[13]; @@ -155,14 +154,14 @@ enum TSQL_TYPE { typedef struct SQuerySQL { struct tSQLExprList *pSelection; // select clause - struct SSQLToken from; // from clause + tVariantList * from; // from clause struct tSQLExpr * pWhere; // where clause [optional] tVariantList * pGroupby; // groupby clause, only for tags[optional] tVariantList * pSortOrder; // orderby [optional] SSQLToken interval; // interval [optional] SSQLToken sliding; // sliding window [optional] SLimitVal limit; // limit offset [optional] - SLimitVal glimit; // group limit offset [optional] + SLimitVal slimit; // group limit offset [optional] tVariantList * fillType; // fill type[optional] SSQLToken selectToken; // sql string } SQuerySQL; @@ -327,12 +326,12 @@ void Parse(void *yyp, int yymajor, ParseTOKENTYPE yyminor, SSqlInfo *); */ void ParseFree(void *p, void (*freeProc)(void *)); -tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToken, uint8_t sortOrder); tVariantList *tVariantListAppend(tVariantList *pList, tVariant *pVar, uint8_t sortOrder); tVariantList *tVariantListInsert(tVariantList *pList, tVariant *pVar, uint8_t sortOrder, int32_t index); -void tVariantListDestroy(tVariantList *pList); +tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToken, uint8_t sortOrder); +void tVariantListDestroy(tVariantList *pList); tFieldList *tFieldListAppend(tFieldList *pList, TAOS_FIELD *pField); @@ -348,12 +347,15 @@ void tSQLExprListDestroy(tSQLExprList *pList); int32_t tSQLSyntaxNodeToString(tSQLExpr *pNode, char *dst); -SQuerySQL *tSetQuerySQLElems(SSQLToken *pSelectToken, tSQLExprList *pSelection, SSQLToken *pFrom, tSQLExpr *pWhere, +SQuerySQL *tSetQuerySQLElems(SSQLToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, tVariantList *pGroupby, tVariantList *pSortOrder, SSQLToken *pInterval, SSQLToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit); SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLToken *pMetricName, tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type); +void tSQLExprDestroy(tSQLExpr *); +void tSQLExprNodeDestroy(tSQLExpr *pExpr); +tSQLExpr *tSQLExprNodeClone(tSQLExpr *pExpr); SAlterTableSQL *tAlterTableSQLElems(SSQLToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type); diff --git a/src/inc/tsqldef.h b/src/inc/tsqldef.h index 86c7c29f30..6d2c166eab 100644 --- a/src/inc/tsqldef.h +++ b/src/inc/tsqldef.h @@ -63,143 +63,155 @@ #define TK_DATABASES 45 #define TK_MNODES 46 #define TK_DNODES 47 -#define TK_USERS 48 -#define TK_MODULES 49 -#define TK_QUERIES 50 -#define TK_CONNECTIONS 51 -#define TK_STREAMS 52 -#define TK_CONFIGS 53 -#define TK_SCORES 54 -#define TK_GRANTS 55 -#define TK_DOT 56 -#define TK_TABLES 57 -#define TK_STABLES 58 -#define TK_VGROUPS 59 -#define TK_DROP 60 -#define TK_TABLE 61 -#define TK_DATABASE 62 -#define TK_USER 63 -#define TK_USE 64 -#define TK_DESCRIBE 65 -#define TK_ALTER 66 -#define TK_PASS 67 -#define TK_PRIVILEGE 68 -#define TK_DNODE 69 -#define TK_IP 70 -#define TK_LOCAL 71 -#define TK_IF 72 -#define TK_EXISTS 73 -#define TK_CREATE 74 -#define TK_KEEP 75 -#define TK_CACHE 76 -#define TK_REPLICA 77 -#define TK_DAYS 78 -#define TK_ROWS 79 -#define TK_ABLOCKS 80 -#define TK_TBLOCKS 81 -#define TK_CTIME 82 -#define TK_CLOG 83 -#define TK_COMP 84 -#define TK_PRECISION 85 -#define TK_LP 86 -#define TK_RP 87 -#define TK_TAGS 88 -#define TK_USING 89 -#define TK_AS 90 -#define TK_COMMA 91 -#define TK_NULL 92 -#define TK_SELECT 93 -#define TK_FROM 94 -#define TK_VARIABLE 95 -#define TK_INTERVAL 96 -#define TK_FILL 97 -#define TK_SLIDING 98 -#define TK_ORDER 99 -#define TK_BY 100 -#define TK_ASC 101 -#define TK_DESC 102 -#define TK_GROUP 103 -#define TK_HAVING 104 -#define TK_LIMIT 105 -#define TK_OFFSET 106 -#define TK_SLIMIT 107 -#define TK_SOFFSET 108 -#define TK_WHERE 109 -#define TK_NOW 110 -#define TK_INSERT 111 -#define TK_INTO 112 -#define TK_VALUES 113 -#define TK_RESET 114 -#define TK_QUERY 115 -#define TK_ADD 116 -#define TK_COLUMN 117 -#define TK_TAG 118 -#define TK_CHANGE 119 -#define TK_SET 120 -#define TK_KILL 121 -#define TK_CONNECTION 122 -#define TK_COLON 123 -#define TK_STREAM 124 -#define TK_ABORT 125 -#define TK_AFTER 126 -#define TK_ATTACH 127 -#define TK_BEFORE 128 -#define TK_BEGIN 129 -#define TK_CASCADE 130 -#define TK_CLUSTER 131 -#define TK_CONFLICT 132 -#define TK_COPY 133 -#define TK_DEFERRED 134 -#define TK_DELIMITERS 135 -#define TK_DETACH 136 -#define TK_EACH 137 -#define TK_END 138 -#define TK_EXPLAIN 139 -#define TK_FAIL 140 -#define TK_FOR 141 -#define TK_IGNORE 142 -#define TK_IMMEDIATE 143 -#define TK_INITIALLY 144 -#define TK_INSTEAD 145 -#define TK_MATCH 146 -#define TK_KEY 147 -#define TK_OF 148 -#define TK_RAISE 149 -#define TK_REPLACE 150 -#define TK_RESTRICT 151 -#define TK_ROW 152 -#define TK_STATEMENT 153 -#define TK_TRIGGER 154 -#define TK_VIEW 155 -#define TK_ALL 156 -#define TK_COUNT 157 -#define TK_SUM 158 -#define TK_AVG 159 -#define TK_MIN 160 -#define TK_MAX 161 -#define TK_FIRST 162 -#define TK_LAST 163 -#define TK_TOP 164 -#define TK_BOTTOM 165 -#define TK_STDDEV 166 -#define TK_PERCENTILE 167 -#define TK_APERCENTILE 168 -#define TK_LEASTSQUARES 169 -#define TK_HISTOGRAM 170 -#define TK_DIFF 171 -#define TK_SPREAD 172 -#define TK_WAVG 173 -#define TK_INTERP 174 -#define TK_LAST_ROW 175 -#define TK_SEMI 176 -#define TK_NONE 177 -#define TK_PREV 178 -#define TK_LINEAR 179 -#define TK_IMPORT 180 -#define TK_METRIC 181 -#define TK_TBNAME 182 -#define TK_JOIN 183 -#define TK_METRICS 184 -#define TK_STABLE 185 +#define TK_ACCOUNTS 48 +#define TK_USERS 49 +#define TK_MODULES 50 +#define TK_QUERIES 51 +#define TK_CONNECTIONS 52 +#define TK_STREAMS 53 +#define TK_CONFIGS 54 +#define TK_SCORES 55 +#define TK_GRANTS 56 +#define TK_DOT 57 +#define TK_TABLES 58 +#define TK_STABLES 59 +#define TK_VGROUPS 60 +#define TK_DROP 61 +#define TK_TABLE 62 +#define TK_DATABASE 63 +#define TK_DNODE 64 +#define TK_IP 65 +#define TK_USER 66 +#define TK_ACCOUNT 67 +#define TK_USE 68 +#define TK_DESCRIBE 69 +#define TK_ALTER 70 +#define TK_PASS 71 +#define TK_PRIVILEGE 72 +#define TK_LOCAL 73 +#define TK_IF 74 +#define TK_EXISTS 75 +#define TK_CREATE 76 +#define TK_PPS 77 +#define TK_TSERIES 78 +#define TK_DBS 79 +#define TK_STORAGE 80 +#define TK_QTIME 81 +#define TK_CONNS 82 +#define TK_STATE 83 +#define TK_KEEP 84 +#define TK_CACHE 85 +#define TK_REPLICA 86 +#define TK_DAYS 87 +#define TK_ROWS 88 +#define TK_ABLOCKS 89 +#define TK_TBLOCKS 90 +#define TK_CTIME 91 +#define TK_CLOG 92 +#define TK_COMP 93 +#define TK_PRECISION 94 +#define TK_LP 95 +#define TK_RP 96 +#define TK_TAGS 97 +#define TK_USING 98 +#define TK_AS 99 +#define TK_COMMA 100 +#define TK_NULL 101 +#define TK_SELECT 102 +#define TK_FROM 103 +#define TK_VARIABLE 104 +#define TK_INTERVAL 105 +#define TK_FILL 106 +#define TK_SLIDING 107 +#define TK_ORDER 108 +#define TK_BY 109 +#define TK_ASC 110 +#define TK_DESC 111 +#define TK_GROUP 112 +#define TK_HAVING 113 +#define TK_LIMIT 114 +#define TK_OFFSET 115 +#define TK_SLIMIT 116 +#define TK_SOFFSET 117 +#define TK_WHERE 118 +#define TK_NOW 119 +#define TK_INSERT 120 +#define TK_INTO 121 +#define TK_VALUES 122 +#define TK_RESET 123 +#define TK_QUERY 124 +#define TK_ADD 125 +#define TK_COLUMN 126 +#define TK_TAG 127 +#define TK_CHANGE 128 +#define TK_SET 129 +#define TK_KILL 130 +#define TK_CONNECTION 131 +#define TK_COLON 132 +#define TK_STREAM 133 +#define TK_ABORT 134 +#define TK_AFTER 135 +#define TK_ATTACH 136 +#define TK_BEFORE 137 +#define TK_BEGIN 138 +#define TK_CASCADE 139 +#define TK_CLUSTER 140 +#define TK_CONFLICT 141 +#define TK_COPY 142 +#define TK_DEFERRED 143 +#define TK_DELIMITERS 144 +#define TK_DETACH 145 +#define TK_EACH 146 +#define TK_END 147 +#define TK_EXPLAIN 148 +#define TK_FAIL 149 +#define TK_FOR 150 +#define TK_IGNORE 151 +#define TK_IMMEDIATE 152 +#define TK_INITIALLY 153 +#define TK_INSTEAD 154 +#define TK_MATCH 155 +#define TK_KEY 156 +#define TK_OF 157 +#define TK_RAISE 158 +#define TK_REPLACE 159 +#define TK_RESTRICT 160 +#define TK_ROW 161 +#define TK_STATEMENT 162 +#define TK_TRIGGER 163 +#define TK_VIEW 164 +#define TK_ALL 165 +#define TK_COUNT 166 +#define TK_SUM 167 +#define TK_AVG 168 +#define TK_MIN 169 +#define TK_MAX 170 +#define TK_FIRST 171 +#define TK_LAST 172 +#define TK_TOP 173 +#define TK_BOTTOM 174 +#define TK_STDDEV 175 +#define TK_PERCENTILE 176 +#define TK_APERCENTILE 177 +#define TK_LEASTSQUARES 178 +#define TK_HISTOGRAM 179 +#define TK_DIFF 180 +#define TK_SPREAD 181 +#define TK_TWA 182 +#define TK_INTERP 183 +#define TK_LAST_ROW 184 +#define TK_SEMI 185 +#define TK_NONE 186 +#define TK_PREV 187 +#define TK_LINEAR 188 +#define TK_IMPORT 189 +#define TK_METRIC 190 +#define TK_TBNAME 191 +#define TK_JOIN 192 +#define TK_METRICS 193 +#define TK_STABLE 194 +#define TK_QUESTION 195 + +#endif + -#endif \ No newline at end of file diff --git a/src/inc/tsqlfunction.h b/src/inc/tsqlfunction.h index 7963dbe829..b3243cd811 100644 --- a/src/inc/tsqlfunction.h +++ b/src/inc/tsqlfunction.h @@ -39,72 +39,80 @@ extern "C" { #define TSDB_FUNC_FIRST 8 #define TSDB_FUNC_LAST 9 #define TSDB_FUNC_LAST_ROW 10 -#define TSDB_FUNC_LEASTSQR 11 -#define TSDB_FUNC_TOP 12 -#define TSDB_FUNC_BOTTOM 13 -#define TSDB_FUNC_SPREAD 14 -#define TSDB_FUNC_WAVG 15 +#define TSDB_FUNC_TOP 11 +#define TSDB_FUNC_BOTTOM 12 +#define TSDB_FUNC_SPREAD 13 +#define TSDB_FUNC_TWA 14 +#define TSDB_FUNC_LEASTSQR 15 + #define TSDB_FUNC_TS 16 #define TSDB_FUNC_TS_DUMMY 17 +#define TSDB_FUNC_TAG_DUMMY 18 +#define TSDB_FUNC_TS_COMP 19 -#define TSDB_FUNC_TAG 18 -#define TSDB_FUNC_PRJ 19 - -#define TSDB_FUNC_TAGPRJ 20 -#define TSDB_FUNC_ARITHM 21 -#define TSDB_FUNC_DIFF 22 +#define TSDB_FUNC_TAG 20 +#define TSDB_FUNC_PRJ 21 -#define TSDB_FUNC_SUM_DST 23 -#define TSDB_FUNC_AVG_DST 24 -#define TSDB_FUNC_MIN_DST 25 -#define TSDB_FUNC_MAX_DST 26 +#define TSDB_FUNC_TAGPRJ 22 +#define TSDB_FUNC_ARITHM 23 +#define TSDB_FUNC_DIFF 24 -#define TSDB_FUNC_FIRST_DST 27 -#define TSDB_FUNC_LAST_DST 28 -#define TSDB_FUNC_LAST_ROW_DST 29 -#define TSDB_FUNC_SPREAD_DST 30 +#define TSDB_FUNC_FIRST_DST 25 +#define TSDB_FUNC_LAST_DST 26 +#define TSDB_FUNC_INTERP 27 -#define TSDB_FUNC_WAVG_DST 31 -#define TSDB_FUNC_TOP_DST 32 -#define TSDB_FUNC_BOTTOM_DST 33 -#define TSDB_FUNC_APERCT_DST 34 -#define TSDB_FUNC_INTERP 35 - -#define TSDB_FUNCSTATE_SO 0x1 // single output -#define TSDB_FUNCSTATE_MO 0x2 // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM -#define TSDB_FUNCSTATE_STREAM 0x4 // function avail for stream -#define TSDB_FUNCSTATE_METRIC 0x8 // function avail for metric -#define TSDB_FUNCSTATE_OF 0x10 // outer forward -#define TSDB_FUNCSTATE_NEED_TS 0x20 +#define TSDB_FUNCSTATE_SO 0x1U // single output +#define TSDB_FUNCSTATE_MO 0x2U // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM +#define TSDB_FUNCSTATE_STREAM 0x4U // function avail for stream +#define TSDB_FUNCSTATE_METRIC 0x8U // function avail for metric +#define TSDB_FUNCSTATE_OF 0x10U // outer forward +#define TSDB_FUNCSTATE_NEED_TS 0x20U // timestamp is required during query processing +#define TSDB_FUNCSTATE_SELECTIVITY 0x40U // selectivity functions, can exists along with tag columns #define TSDB_BASE_FUNC_SO TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF #define TSDB_BASE_FUNC_MO TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF -#define TSDB_PATTERN_MATCH 0 -#define TSDB_PATTERN_NOMATCH 1 -#define TSDB_PATTERN_NOWILDCARDMATCH 2 -#define TSDB_PATTERN_STRING_MAX_LEN 20 +#define TSDB_PATTERN_MATCH 0 +#define TSDB_PATTERN_NOMATCH 1 +#define TSDB_PATTERN_NOWILDCARDMATCH 2 +#define TSDB_PATTERN_STRING_MAX_LEN 20 -#define TSDB_FUNCTIONS_NAME_MAX_LENGTH 16 +#define TSDB_FUNCTIONS_NAME_MAX_LENGTH 16 #define TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE 50 #define PATTERN_COMPARE_INFO_INITIALIZER \ { '%', '_' } #define DATA_SET_FLAG ',' // to denote the output area has data, not null value -#define DATA_SET_FLAG_SIZE sizeof(char) +#define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG) + +#define QUERY_COND_REL_PREFIX_IN "IN|" +#define QUERY_COND_REL_PREFIX_LIKE "LIKE|" -#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_COND_REL_PREFIX_IN_LEN 3 +#define QUERY_COND_REL_PREFIX_LIKE_LEN 5 + +#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 + #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSQL_SO_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) +#define MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY 10000000 +#define TOP_BOTTOM_QUERY_LIMIT 100 + enum { - MASTER_SCAN = 0x0, - SUPPLEMENTARY_SCAN = 0x1, - SECONDARY_STAGE_MERGE = 0x10, + MASTER_SCAN = 0x0, + SUPPLEMENTARY_SCAN = 0x1, + FIRST_STAGE_MERGE = 0x10, + SECONDARY_STAGE_MERGE = 0x20, }; -typedef struct { +#define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0) +#define QUERY_IS_JOIN_QUERY(type) (((type)&TSDB_QUERY_TYPE_JOIN_QUERY) != 0) +#define QUERY_IS_PROJECTION_QUERY(type) (((type)&TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) +#define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0) + +typedef struct SArithmeticSupport { SSqlFunctionExpr *pExpr; int32_t elemSize[TSDB_MAX_COLUMNS]; int32_t numOfCols; @@ -114,81 +122,92 @@ typedef struct { typedef struct SQLPreAggVal { bool isSet; - int32_t numOfNullPoints; - int64_t wsum; + int32_t numOfNull; int64_t sum; - int64_t min; int64_t max; + int64_t min; + int16_t maxIndex; + int16_t minIndex; } SQLPreAggVal; -/* sql function runtime context */ +typedef struct SInterpInfoDetail { + TSKEY ts; // interp specified timestamp + int8_t hasResult; + int8_t type; + int8_t primaryCol; +} SInterpInfoDetail; + +typedef struct SInterpInfo { SInterpInfoDetail *pInterpDetail; } SInterpInfo; + +typedef struct SResultInfo { + int8_t hasResult; // result generated, not NULL value + bool initialized; // output buffer has been initialized + bool complete; // query has completed + bool superTableQ; // is super table query + int32_t numOfRes; // num of output result in current buffer + int32_t bufLen; // buffer size + void * interResultBuf; // output result buffer +} SResultInfo; + +struct SQLFunctionCtx; + +/** + * for selectivity query, the corresponding tag value is assigned if the data is qualified + */ +typedef struct SExtTagsInfo { + int16_t tagsLen; // keep the tags data for top/bottom query result + int16_t numOfTagCols; + struct SQLFunctionCtx **pTagCtxList; +} SExtTagsInfo; + +// sql function runtime context typedef struct SQLFunctionCtx { int32_t startOffset; int32_t size; int32_t order; - int32_t scanFlag; + int32_t scanFlag; // TODO merge with currentStage int16_t inputType; int16_t inputBytes; - int16_t outputType; - int16_t outputBytes; /* size of results, determined by function and input - column data type */ - - bool hasNullValue; /* null value exist in current block */ - int32_t blockStatus; /* Indicate if data is loaded, it is first/last/internal - block. Only for file blocks */ - - void * aInputElemBuf; - char * aOutputBuf; /* final result output buffer, point to sdata->data */ - int64_t numOfIteratedElems; /* total scanned points in processing, used for - complex query process */ - int32_t numOfOutputElems; - - int32_t currentStage; /* record current running step, default: 0 */ - - int64_t nStartQueryTimestamp; /* timestamp range of current query when - function is executed on a specific data block - */ - tVariant intermediateBuf[4]; /* to hold intermediate result */ - + int16_t outputType; + int16_t outputBytes; // size of results, determined by function and input column data type + bool hasNull; // null value exist in current block + int16_t functionId; // function id + int32_t blockStatus; // Indicate if data is loaded, it is first/last/internal block. Only for file blocks + void * aInputElemBuf; + char * aOutputBuf; // final result output buffer, point to sdata->data + uint8_t currentStage; // record current running step, default: 0 + int64_t nStartQueryTimestamp; // timestamp range of current query when function is executed on a specific data block int32_t numOfParams; - tVariant param[4]; /* input parameter, current support only one element */ - int64_t *ptsList; /* additional array list */ - void * ptsOutputBuf; /* output buffer for the corresponding timestamp of each - result, e.g., top/bottom*/ - + tVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param */ + int64_t *ptsList; // corresponding timestamp array list + void * ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ SQLPreAggVal preAggVals; + tVariant tag; + SResultInfo *resultInfo; + + SExtTagsInfo tagInfo; } SQLFunctionCtx; typedef struct SQLAggFuncElem { char aName[TSDB_FUNCTIONS_NAME_MAX_LENGTH]; - uint8_t nAggIdx; /* index of function in aAggs */ - int8_t stableFuncId; /* transfer function for metric query */ + uint8_t nAggIdx; // index of function in aAggs + int8_t stableFuncId; // transfer function for super table query uint16_t nStatus; - /* setup the execute environment */ - void (*init)(SQLFunctionCtx *pCtx); + bool (*init)(SQLFunctionCtx *pCtx); // setup the execute environment - /* main execution function */ - bool (*xFunction)(SQLFunctionCtx *pCtx); + void (*xFunction)(SQLFunctionCtx *pCtx); // blocks version function + void (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position); // single-row function version - /* filter version */ - bool (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position); - - /* - * some sql function require scan data twice or more in case of no index - * existing. - * e.g., stddev, percentile[disk based process for extremely large dataset] - * @param pCtx - */ - bool (*xNextStep)(SQLFunctionCtx *pCtx); + // some sql function require scan data twice or more, e.g.,stddev + void (*xNextStep)(SQLFunctionCtx *pCtx); /* * finalizer must be called after all xFunction has been executed to - * generated final result. Otherwise, the value in aOutputBuf is a intern - * result. + * generated final result. Otherwise, the value in aOutputBuf is a intern result. */ void (*xFinalize)(SQLFunctionCtx *pCtx); @@ -204,24 +223,25 @@ typedef struct SPatternCompareInfo { char matchOne; // symbol for match one wildcard, default: '_' } SPatternCompareInfo; -void function_finalize(SQLFunctionCtx *pCtx); +#define GET_RES_INFO(ctx) ((ctx)->resultInfo) + +int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, + int16_t *len, int16_t *interResBytes, int16_t extLength, bool isSuperTable); -void getResultInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, int16_t *len); +SResultInfo *getResultSupportInfo(SQLFunctionCtx *pCtx); int patternMatch(const char *zPattern, const char *zString, size_t size, const SPatternCompareInfo *pInfo); -int WCSPatternMatch(const wchar_t *zPattern, const wchar_t *zString, size_t size, - const struct SPatternCompareInfo *pInfo); +int WCSPatternMatch(const wchar_t *zPattern, const wchar_t *zString, size_t size, const SPatternCompareInfo *pInfo); -#define IS_STREAM_QUERY_VALID(x) (((x)&TSDB_FUNCSTATE_STREAM) != 0) -#define IS_MULTIOUTPUT(x) (((x)&TSDB_FUNCSTATE_MO) != 0) -#define IS_SINGLEOUTPUT(x) (((x)&TSDB_FUNCSTATE_SO) != 0) -#define IS_OUTER_FORWARD(x) (((x)&TSDB_FUNCSTATE_OF) != 0) +#define IS_STREAM_QUERY_VALID(x) (((x)&TSDB_FUNCSTATE_STREAM) != 0) +#define IS_MULTIOUTPUT(x) (((x)&TSDB_FUNCSTATE_MO) != 0) +#define IS_SINGLEOUTPUT(x) (((x)&TSDB_FUNCSTATE_SO) != 0) +#define IS_OUTER_FORWARD(x) (((x)&TSDB_FUNCSTATE_OF) != 0) /* * the status of one block, used in metric query. all blocks are mixed together, - * we need the status to decide - * if one block is a first/end/inter block of one meter + * we need the status to decide if one block is a first/end/inter block of one meter */ enum { BLK_FILE_BLOCK = 0x1, @@ -251,12 +271,12 @@ enum { #define SET_DATA_BLOCK_LOADED(x) ((x) |= BLK_BLOCK_LOADED); #define IS_DATA_BLOCK_LOADED(x) (((x)&BLK_BLOCK_LOADED) != 0) -typedef struct SWavgRuntime { - int8_t valFlag; // flag to denote has value - int16_t type; // source data type - int64_t lastKey; - int64_t sKey; - int64_t eKey; +typedef struct STwaInfo { + TSKEY lastKey; + int8_t hasResult; // flag to denote has value + int16_t type; // source data type + TSKEY SKey; + TSKEY EKey; union { double dOutput; @@ -267,33 +287,23 @@ typedef struct SWavgRuntime { double dLastValue; int64_t iLastValue; }; -} SWavgRuntime; - -typedef struct SSumRuntime { - union { - double dOutput; - int64_t iOutput; - }; - int8_t valFlag; -} SSumRuntime; - -typedef struct SAvgRuntime { - double sum; - int64_t num; - int8_t valFlag; -} SAvgRuntime; +} STwaInfo; /* global sql function array */ -extern struct SQLAggFuncElem aAggs[36]; +extern struct SQLAggFuncElem aAggs[28]; /* compatible check array list */ -extern int32_t funcCompatList[36]; +extern int32_t funcCompatDefList[28]; void getStatistics(char *priData, char *data, int32_t size, int32_t numOfRow, int32_t type, int64_t *min, int64_t *max, - int64_t *sum, int64_t *wsum, int32_t *numOfNull); + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull); bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *minval, char *maxval); +void resetResultInfo(SResultInfo *pResInfo); +void initResultInfo(SResultInfo *pResInfo); +void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable); + #ifdef __cplusplus } #endif diff --git a/src/inc/tstatus.h b/src/inc/tstatus.h index 7b2ff4a7dc..34bc7c6f27 100644 --- a/src/inc/tstatus.h +++ b/src/inc/tstatus.h @@ -22,6 +22,8 @@ extern "C" { extern char *sdbDnodeStatusStr[]; extern char *sdbDnodeBalanceStateStr[]; +extern char *sdbVnodeDropStateStr[]; +extern char *sdbVnodeSyncStatusStr[]; #ifdef __cplusplus } diff --git a/src/inc/tstoken.h b/src/inc/tstoken.h index 2e508ab06d..420fdaad9b 100644 --- a/src/inc/tstoken.h +++ b/src/inc/tstoken.h @@ -29,8 +29,9 @@ typedef struct SSQLToken { char * z; } SSQLToken; +#if 0 char *tscGetToken(char *string, char **token, int *tokenLen); -char *tscGetTokenDelimiter(char *string, char **token, int *tokenLen, const char *delimiters); +#endif /** * tokenizer for sql string @@ -40,14 +41,39 @@ char *tscGetTokenDelimiter(char *string, char **token, int *tokenLen, const char */ uint32_t tSQLGetToken(char *z, uint32_t *tokenType); -void tStrGetToken(char *str, int32_t *i, SSQLToken *t0, bool isPrevOptr); +/** + * enhanced tokenizer for sql string. + * + * @param str + * @param i + * @param isPrevOptr + * @param numOfIgnoreToken + * @param ignoreTokenTypes + * @return + */ +SSQLToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t *ignoreTokenTypes); +/** + * check if it is a keyword or not + * @param z + * @param len + * @return + */ bool isKeyWord(const char *z, int32_t len); -bool isNumber(const SSQLToken *pToken); -void shiftStr(char *dst, char *src); +/** + * check if it is a number or not + * @param pToken + * @return + */ +bool isNumber(const SSQLToken *pToken); -uint64_t changeToTimestampWithDynPrec(SSQLToken *pToken); +/** + * check if it is a token or not + * @param pToken + * @return token type, if it is not a number, TK_ILLEGAL will return + */ +int32_t isValidNumber(const SSQLToken* pToken); #ifdef __cplusplus } diff --git a/src/inc/tstrbuild.h b/src/inc/tstrbuild.h new file mode 100644 index 0000000000..68d1914be3 --- /dev/null +++ b/src/inc/tstrbuild.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_STRING_BUILDER_H +#define TDENGINE_STRING_BUILDER_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SStringBuilder { + jmp_buf jb; + size_t size; + size_t pos; + char* buf; +} SStringBuilder; + +#define taosStringBuilderSetJmp(sb) setjmp((sb)->jb) + +void taosStringBuilderEnsureCapacity(SStringBuilder* sb, size_t size); +char* taosStringBuilderGetResult(SStringBuilder* sb, size_t* len); +void taosStringBuilderDestroy(SStringBuilder* sb); + +void taosStringBuilderAppend(SStringBuilder* sb, const void* data, size_t len); +void taosStringBuilderAppendChar(SStringBuilder* sb, char c); +void taosStringBuilderAppendStringLen(SStringBuilder* sb, const char* str, size_t len); +void taosStringBuilderAppendString(SStringBuilder* sb, const char* str); +void taosStringBuilderAppendNull(SStringBuilder* sb); +void taosStringBuilderAppendInteger(SStringBuilder* sb, int64_t v); +void taosStringBuilderAppendDouble(SStringBuilder* sb, double v); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc/tsystem.h b/src/inc/tsystem.h index 832956ae80..9261261e1b 100644 --- a/src/inc/tsystem.h +++ b/src/inc/tsystem.h @@ -23,6 +23,8 @@ extern "C" { #include #include +extern char dataDir[TSDB_FILENAME_LEN]; + bool taosGetSysMemory(float *memoryUsedMB); bool taosGetProcMemory(float *memoryUsedMB); diff --git a/src/inc/ttimer.h b/src/inc/ttimer.h index 1f41a1ac76..8a7e81dac7 100644 --- a/src/inc/ttimer.h +++ b/src/inc/ttimer.h @@ -45,6 +45,8 @@ void *taosTmrInit(int maxTmr, int resoultion, int longest, char *label); tmr_h taosTmrStart(void (*fp)(void *, void *), int mseconds, void *param1, void *handle); +void taosTmrStop(tmr_h tmrId); + void taosTmrStopA(tmr_h *timerId); void taosTmrReset(void (*fp)(void *, void *), int mseconds, void *param1, void *handle, tmr_h *pTmrId); diff --git a/src/inc/ttypes.h b/src/inc/ttypes.h index b0f8044404..b2ea8e918a 100644 --- a/src/inc/ttypes.h +++ b/src/inc/ttypes.h @@ -67,9 +67,9 @@ typedef struct tVariant { void tVariantCreate(tVariant *pVar, SSQLToken *token); -void tVariantCreateN(tVariant *pVar, char *pz, uint32_t len, uint32_t type); +void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t type); -void tVariantCreateB(tVariant *pVar, char *pz, uint32_t len, uint32_t type); +void tVariantCreateFromBinary(tVariant *pVar, char *pz, uint32_t len, uint32_t type); void tVariantDestroy(tVariant *pV); diff --git a/src/inc/tutil.h b/src/inc/tutil.h index f6521da99c..f63f8b6cec 100644 --- a/src/inc/tutil.h +++ b/src/inc/tutil.h @@ -16,26 +16,20 @@ #ifndef TDENGINE_TUTIL_H #define TDENGINE_TUTIL_H -#include "os.h" -#include "tmd5.h" - #ifdef __cplusplus extern "C" { #endif -#include -#include -#include -#include -#include -#include -#include -#include - +#include "os.h" +#include "tmd5.h" #include "tcrc32c.h" #include "tsdb.h" -#define VALIDFD(x) ((x) > 2) +#ifndef STDERR_FILENO + #define VALIDFD(x) ((x) > 2) +#else + #define VALIDFD(x) ((x) > STDERR_FILENO) +#endif #define WCHAR wchar_t #define tfree(x) \ @@ -101,8 +95,24 @@ extern "C" { #define GET_INT16_VAL(x) (*(int16_t *)(x)) #define GET_INT32_VAL(x) (*(int32_t *)(x)) #define GET_INT64_VAL(x) (*(int64_t *)(x)) -#define GET_FLOAT_VAL(x) (*(float *)(x)) -#define GET_DOUBLE_VAL(x) (*(double *)(x)) + +#ifdef _TD_ARM_32_ + #define GET_FLOAT_VAL(x) taos_align_get_float(x) + #define GET_DOUBLE_VAL(x) taos_align_get_double(x) + + float taos_align_get_float(char* pBuf); + double taos_align_get_double(char* pBuf); + + //#define __float_align_declear() float __underlyFloat = 0.0; + //#define __float_align_declear() + //#define GET_FLOAT_VAL_ALIGN(x) (*(int32_t*)&(__underlyFloat) = *(int32_t*)(x); __underlyFloat); + // notes: src must be float or double type variable !!! + #define SET_FLOAT_VAL_ALIGN(dst, src) (*(int32_t*) dst = *(int32_t*)src); + #define SET_DOUBLE_VAL_ALIGN(dst, src) (*(int64_t*) dst = *(int64_t*)src); +#else + #define GET_FLOAT_VAL(x) (*(float *)(x)) + #define GET_DOUBLE_VAL(x) (*(double *)(x)) +#endif #define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1))) @@ -132,6 +142,10 @@ int64_t strnatoi(char *num, int32_t len); char* strreplace(const char* str, const char* pattern, const char* rep); +#define POW2(x) ((x) * (x)) + +int32_t strdequote(char *src); + char *paGetToken(char *src, char **token, int32_t *tokenLen); void taosMsleep(int32_t mseconds); @@ -173,6 +187,21 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, cha memcpy(target, context.digest, TSDB_KEY_LEN); } +char *taosIpStr(uint32_t ipInt); + +#ifdef _TAOS_MEM_TEST_ +// Use during test to simulate the success and failure scenarios of memory allocation +extern void* taos_malloc(unsigned int size, char* _func); +extern void* taos_calloc(unsigned int num, unsigned int size, char* _func); +extern void* taos_realloc(void* ptr, unsigned int size, char* _func); +extern void taos_free(void* ptr); +#define malloc(size) taos_malloc(size, __FUNCTION__) +#define calloc(num, size) taos_calloc(num, size, __FUNCTION__) +#define realloc(ptr, size) taos_realloc(ptr, size, __FUNCTION__) +#define free(ptr) taos_free(ptr) +#endif + + #ifdef __cplusplus } #endif diff --git a/src/kit/CMakeLists.txt b/src/kit/CMakeLists.txt index 259958675b..66e8cf7398 100644 --- a/src/kit/CMakeLists.txt +++ b/src/kit/CMakeLists.txt @@ -1,5 +1,4 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) ADD_SUBDIRECTORY(shell) diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index f6400c7e5b..20aa0c9d31 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -1,43 +1,32 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -IF (TD_LINUX) - - INCLUDE_DIRECTORIES(inc ${TD_ROOT_DIR}/src/inc ${TD_ROOT_DIR}/src/client/inc ${TD_OS_DIR}/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) +INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) +INCLUDE_DIRECTORIES(inc) +IF (TD_LINUX_64) AUX_SOURCE_DIRECTORY(./src SRC) LIST(REMOVE_ITEM SRC ./src/shellWindows.c) - ADD_EXECUTABLE(shell ${SRC}) TARGET_LINK_LIBRARIES(shell taos_static) - SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME "taos") - -ELSEIF (TD_WINDOWS) - - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/deps/pthread) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/deps/regex) - INCLUDE_DIRECTORIES(inc ${TD_ROOT_DIR}/src/inc ${TD_ROOT_DIR}/src/client/inc ${TD_OS_DIR}/inc) - + SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) +ELSEIF (TD_WINDOWS_64) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/regex) LIST(APPEND SRC ./src/shellEngine.c) LIST(APPEND SRC ./src/shellMain.c) LIST(APPEND SRC ./src/shellWindows.c) - ADD_EXECUTABLE(shell ${SRC}) TARGET_LINK_LIBRARIES(shell taos_static) - SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME "taos") - -ELSEIF (TD_DARWIN) - - INCLUDE_DIRECTORIES(inc ${TD_ROOT_DIR}/src/inc ${TD_ROOT_DIR}/src/client/inc ${TD_OS_DIR}/inc) - + SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) +ELSEIF (TD_DARWIN_64) LIST(APPEND SRC ./src/shellEngine.c) LIST(APPEND SRC ./src/shellMain.c) LIST(APPEND SRC ./src/shellWindows.c) - ADD_EXECUTABLE(shell ${SRC}) TARGET_LINK_LIBRARIES(shell taos_static) - SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME "taos") - + SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ENDIF () diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h index eca04b31c2..12024d0b86 100644 --- a/src/kit/shell/inc/shell.h +++ b/src/kit/shell/inc/shell.h @@ -20,6 +20,7 @@ #include "taos.h" #include "tlog.h" #include "tsdb.h" +#include "stdbool.h" #define MAX_USERNAME_SIZE 64 #define MAX_DBNAME_SIZE 64 @@ -78,6 +79,7 @@ void cleanup_handler(void* arg); void exitShell(); int shellDumpResult(TAOS* con, char* fname, int* error_no, bool printMode); void shellPrintNChar(char* str, int width, bool printMode); +void shellGetGrantInfo(void *con); #define max(a, b) ((int)(a) < (int)(b) ? (int)(b) : (int)(a)) /**************** Global variable declarations ****************/ diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 3cf77bd73b..324dde55d1 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -443,17 +443,27 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { printf("%*d|", l[i], *((int *)row[i])); break; case TSDB_DATA_TYPE_BIGINT: -#ifdef LINUX - printf("%*ld|", l[i], *((int64_t *)row[i])); -#else printf("%*lld|", l[i], *((int64_t *)row[i])); -#endif break; case TSDB_DATA_TYPE_FLOAT: +#ifdef _TD_ARM_32_ + float fv = 0; + //memcpy(&fv, row[i], sizeof(float)); + *(int32_t*)(&fv) = *(int32_t*)row[i]; + printf("%*.5f|", l[i], fv); +#else printf("%*.5f|", l[i], *((float *)row[i])); +#endif break; case TSDB_DATA_TYPE_DOUBLE: +#ifdef _TD_ARM_32_ + double dv = 0; + //memcpy(&dv, row[i], sizeof(double)); + *(int64_t*)(&dv) = *(int64_t*)row[i]; + printf("%*.9f|", l[i], dv); +#else printf("%*.9f|", l[i], *((double *)row[i])); +#endif break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: @@ -466,16 +476,12 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { break; case TSDB_DATA_TYPE_TIMESTAMP: if (args.is_raw_time) { -#ifdef LINUX - printf(" %ld|", *(int64_t *)row[i]); -#else printf(" %lld|", *(int64_t *)row[i]); -#endif } else { if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) { - tt = *(int64_t *)row[i] / 1000000; + tt = (time_t)((*(int64_t *)row[i]) / 1000000); } else { - tt = *(int64_t *)row[i] / 1000; + tt = (time_t)((*(int64_t *)row[i]) / 1000); } ptm = localtime(&tt); @@ -520,18 +526,28 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { printf("%d\n", *((int *)row[i])); break; case TSDB_DATA_TYPE_BIGINT: -#ifdef LINUX - printf("%ld\n", *((int64_t *)row[i])); -#else printf("%lld\n", *((int64_t *)row[i])); -#endif break; case TSDB_DATA_TYPE_FLOAT: +#ifdef _TD_ARM_32_ + float fv = 0; + //memcpy(&fv, row[i], sizeof(float)); + *(int32_t*)(&fv) = *(int32_t*)row[i]; + printf("%.5f\n", fv); +#else printf("%.5f\n", *((float *)row[i])); +#endif break; case TSDB_DATA_TYPE_DOUBLE: +#ifdef _TD_ARM_32_ + double dv = 0; + //memcpy(&dv, row[i], sizeof(double)); + *(int64_t*)(&dv) = *(int64_t*)row[i]; + printf("%.9f\n", dv); +#else printf("%.9f\n", *((double *)row[i])); - break; +#endif + break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: memset(t_str, 0, TSDB_MAX_BYTES_PER_ROW); @@ -541,16 +557,12 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { break; case TSDB_DATA_TYPE_TIMESTAMP: if (args.is_raw_time) { -#ifdef LINUX - printf("%ld\n", *(int64_t *)row[i]); -#else printf("%lld\n", *(int64_t *)row[i]); -#endif } else { if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) { - tt = *(int64_t *)row[i] / 1000000; + tt = (time_t)((*(int64_t *)row[i]) / 1000000); } else { - tt = *(int64_t *)row[i] / 1000; + tt = (time_t)((*(int64_t *)row[i]) / 1000); } ptm = localtime(&tt); @@ -573,6 +585,16 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { } while ((row = taos_fetch_row(result))); } else { // dump to file + // first write column + for (int col = 0; col < num_fields; col++) { + fprintf(fp, "%s", fields[col].name); + if (col < num_fields - 1) { + fprintf(fp, ","); + } else { + fprintf(fp, "\n"); + } + } + do { for (int i = 0; i < num_fields; i++) { if (row[i]) { @@ -590,17 +612,27 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { fprintf(fp, "%d", *((int *)row[i])); break; case TSDB_DATA_TYPE_BIGINT: -#ifdef LINUX - fprintf(fp, "%ld", *((int64_t *)row[i])); -#else fprintf(fp, "%lld", *((int64_t *)row[i])); -#endif break; case TSDB_DATA_TYPE_FLOAT: +#ifdef _TD_ARM_32_ + float fv = 0; + //memcpy(&fv, row[i], sizeof(float)); + *(int32_t*)(&fv) = *(int32_t*)row[i]; + fprintf(fp, "%.5f", fv); +#else fprintf(fp, "%.5f", *((float *)row[i])); +#endif break; case TSDB_DATA_TYPE_DOUBLE: +#ifdef _TD_ARM_32_ + double dv = 0; + //memcpy(&dv, row[i], sizeof(double)); + *(int64_t*)(&dv) = *(int64_t*)row[i]; + fprintf(fp, "%.9f", dv); +#else fprintf(fp, "%.9f", *((double *)row[i])); +#endif break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: @@ -609,11 +641,24 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { fprintf(fp, "\'%s\'", t_str); break; case TSDB_DATA_TYPE_TIMESTAMP: -#ifdef LINUX - fprintf(fp, "%ld", *(int64_t *)row[i]); -#else - fprintf(fp, "%lld", *(int64_t *)row[i]); -#endif + if (args.is_raw_time) { + fprintf(fp, "%lld", *(int64_t *)row[i]); + } else { + if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) { + tt = (time_t)((*(int64_t *)row[i]) / 1000000); + } else { + tt = (time_t)((*(int64_t *)row[i]) / 1000); + } + + ptm = localtime(&tt); + strftime(buf, 64, "%Y-%m-%d %H:%M:%S", ptm); + + if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) { + fprintf(fp, "\'%s.%06d\'", buf, (int)(*(int64_t *)row[i] % 1000000)); + } else { + fprintf(fp, "\'%s.%03d\'", buf, (int)(*(int64_t *)row[i] % 1000)); + } + } break; default: break; @@ -703,7 +748,7 @@ void write_history() { } void taos_error(TAOS *con) { - fprintf(stderr, "\nTSDB error: %s\n\n", taos_errstr(con)); + fprintf(stderr, "\nDB error: %s\n", taos_errstr(con)); /* free local resouce: allocated memory/metric-meta refcnt */ TAOS_RES *pRes = taos_use_result(con); @@ -771,3 +816,52 @@ void source_file(TAOS *con, char *fptr) { wordfree(&full_path); fclose(f); } + +void shellGetGrantInfo(void *con) { +#ifdef CLUSTER + char sql[] = "show grants"; + + if (taos_query(con, sql)) { + fprintf(stdout, "\n"); + return; + } + + int num_fields = taos_field_count(con); + if (num_fields == 0) { + fprintf(stderr, "\nInvalid grant information.\n"); + exit(0); + } else { + result = taos_use_result(con); + if (result == NULL) { + fprintf(stderr, "\nGrant information is null.\n"); + exit(0); + } + + TAOS_FIELD *fields = taos_fetch_fields(result); + TAOS_ROW row = taos_fetch_row(result); + if (row == NULL) { + fprintf(stderr, "\nGrant information is empty.\n"); + exit(0); + } + + char version[32] = {0}; + char expiretime[32] = {0}; + char expired[32] = {0}; + + memcpy(version, row[0], fields[0].bytes); + memcpy(expiretime, row[1], fields[1].bytes); + memcpy(expired, row[2], fields[2].bytes); + + if (strcmp(expiretime, "unlimited") == 0) { + fprintf(stdout, "This is the %s version and will never expire.\n", version); + } else { + fprintf(stdout, "This is the %s version and will expire at %s.\n", version, expiretime); + } + + taos_free_result(result); + result = NULL; + } + + fprintf(stdout, "\n"); +#endif +} diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 193770d1d7..145025cbe1 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -56,7 +56,7 @@ static struct argp_option options[] = { {"user", 'u', "USER", 0, "The TDEngine user name to use when connecting to the server."}, {"config-dir", 'c', "CONFIG_DIR", 0, "Configuration directory."}, {"commands", 's', "COMMANDS", 0, "Commands to run without enter the shell."}, - {"raw-time", 'r', 0, 0, "Output time as unsigned long."}, + {"raw-time", 'r', 0, 0, "Output time as uint64_t."}, {"file", 'f', "FILE", 0, "Script to run without enter the shell."}, {"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."}, {"timezone", 't', "TIMEZONE", 0, "Time zone of the shell, default is local."}, @@ -313,7 +313,7 @@ void shellPrintNChar(char *str, int width, bool printMode) { if (*str == '\0') break; char *tstr = str; int byte_width = mbtowc(&wc, tstr, MB_CUR_MAX); - if (byte_width <= 0 ) break; + if (byte_width <= 0) break; int col_width = wcwidth(wc); if (col_width <= 0) { str += byte_width; diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 8ca51f3935..63c9eac0db 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -96,6 +96,9 @@ int main(int argc, char* argv[]) { sigaction(SIGTERM, &act, NULL); sigaction(SIGINT, &act, NULL); + /* Get grant information */ + shellGetGrantInfo(con); + /* Loop to query the input. */ while (1) { pthread_create(&pid, NULL, shellLoopQuery, con); diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index bffee5390b..61e6bcaf30 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -17,7 +17,7 @@ void printHelp() { char indent[10] = " "; - printf("taos shell is used to test the TAOS database\n"); + printf("taos shell is used to test the TDEngine database\n"); printf("%s%s\n", indent, "-h"); printf("%s%s%s\n", indent, indent, "TDEngine server IP address to connect. The default host is localhost."); @@ -200,7 +200,8 @@ void shellReadCommand(TAOS *con, char command[]) { void *shellLoopQuery(void *arg) { TAOS *con = (TAOS *)arg; - char command[MAX_COMMAND_SIZE]; + char *command = malloc(MAX_COMMAND_SIZE); + if (command == NULL) return NULL; while (1) { memset(command, 0, MAX_COMMAND_SIZE); @@ -244,4 +245,4 @@ void shellPrintNChar(char *str, int width, bool printMode) { void get_history_path(char *history) { sprintf(history, "%s/%s", ".", HISTORY_FILE); } -void exitShell() { exit(EXIT_SUCCESS); } \ No newline at end of file +void exitShell() { exit(EXIT_SUCCESS); } diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt index 727ba94479..38a28e3079 100644 --- a/src/kit/taosdemo/CMakeLists.txt +++ b/src/kit/taosdemo/CMakeLists.txt @@ -1,10 +1,13 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -IF (TD_LINUX) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) +INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) +INCLUDE_DIRECTORIES(inc) + +IF (TD_LINUX_64) AUX_SOURCE_DIRECTORY(. SRC) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/src/inc ${TD_ROOT_DIR}/src/client/inc ${TD_OS_DIR}/inc) ADD_EXECUTABLE(taosdemo ${SRC}) TARGET_LINK_LIBRARIES(taosdemo taos_static) ENDIF () diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 86882368cc..df29a72e86 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -53,10 +53,10 @@ static struct argp_option options[] = { {0, 'b', "type_of_cols", 0, "The data_type of columns: 'INT', 'TINYINT', 'SMALLINT', 'BIGINT', 'FLOAT', 'DOUBLE', 'BINARY'. Default is 'INT'.", 7}, {0, 'w', "length_of_binary", 0, "The length of data_type 'BINARY'. Only applicable when type of cols is 'BINARY'. Default is 8", 8}, {0, 'l', "num_of_cols_per_record", 0, "The number of columns per record. Default is 3.", 8}, - {0, 'c', "num_of_conns", 0, "The number of connections. Default is 10.", 9}, - {0, 'r', "num_of_records_per_req", 0, "The number of records per request. Default is 1000.", 10}, - {0, 't', "num_of_tables", 0, "The number of tables. Default is 10000.", 11}, - {0, 'n', "num_of_records_per_table", 0, "The number of records per table. Default is 100000.", 12}, + {0, 'c', "num_of_conns", 0, "The number of connections. Default is 10.", 9}, + {0, 'r', "num_of_records_per_req", 0, "The number of records per request. Default is 1000.", 10}, + {0, 't', "num_of_tables", 0, "The number of tables. Default is 10000.", 11}, + {0, 'n', "num_of_records_per_table", 0, "The number of records per table. Default is 100000.", 12}, {0, 'f', "config_directory", 0, "Configuration directory. Default is '/etc/taos/'.", 14}, {0, 'x', 0, 0, "Insert only flag.", 13}, {0}}; @@ -212,7 +212,7 @@ typedef struct { int ncols_per_record; int nrecords_per_table; int nrecords_per_request; - long start_time; + int64_t start_time; bool do_aggreFunc; sem_t mutex_sem; @@ -224,7 +224,7 @@ typedef struct { TAOS *taos; char tb_name[MAX_TB_NAME_SIZE]; - long timestamp; + int64_t timestamp; int target; int counter; int nrecords_per_request; @@ -255,7 +255,7 @@ void *syncWrite(void *sarg); void *asyncWrite(void *sarg); -void generateData(char *res, char **data_type, int num_of_cols, long timestamp, int len_of_binary); +void generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary); void rand_string(char *str, int size); @@ -536,7 +536,7 @@ void *readTable(void *sarg) { info *rinfo = (info *)sarg; TAOS *taos = rinfo->taos; char command[BUFFER_SIZE] = "\0"; - long sTime = rinfo->start_time; + int64_t sTime = rinfo->start_time; char *tb_prefix = rinfo->tb_prefix; FILE *fp = fopen(rinfo->fp, "a"); int num_of_DPT = rinfo->nrecords_per_table; @@ -681,11 +681,11 @@ void *syncWrite(void *sarg) { int len_of_binary = winfo->len_of_binary; int ncols_per_record = winfo->ncols_per_record; srand(time(NULL)); - long time_counter = winfo->start_time; + int64_t time_counter = winfo->start_time; for (int i = 0; i < winfo->nrecords_per_table;) { for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { int inserted = i; - long tmp_time = time_counter; + int64_t tmp_time = time_counter; char *pstr = buffer; pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, tID); @@ -749,7 +749,7 @@ void callBack(void *param, TAOS_RES *res, int code) { char **datatype = tb_info->data_type; int ncols_per_record = tb_info->ncols_per_record; int len_of_binary = tb_info->len_of_binary; - long tmp_time = tb_info->timestamp; + int64_t tmp_time = tb_info->timestamp; if (code < 0) { fprintf(stderr, "failed to insert data %d:reason; %s\n", code, taos_errstr(tb_info->taos)); @@ -795,7 +795,7 @@ double getCurrentTime() { return tv.tv_sec + tv.tv_usec / 1E6; } -void generateData(char *res, char **data_type, int num_of_cols, long timestamp, int len_of_binary) { +void generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary) { memset(res, 0, MAX_DATA_SIZE); char *pstr = res; pstr += sprintf(pstr, "(%ld", timestamp); diff --git a/src/kit/taosdump/CMakeLists.txt b/src/kit/taosdump/CMakeLists.txt index 06f6a7df6e..2bcc020654 100644 --- a/src/kit/taosdump/CMakeLists.txt +++ b/src/kit/taosdump/CMakeLists.txt @@ -1,10 +1,13 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -IF (TD_LINUX) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) +INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) +INCLUDE_DIRECTORIES(inc) + +IF (TD_LINUX_64) AUX_SOURCE_DIRECTORY(. SRC) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/src/inc ${TD_ROOT_DIR}/src/client/inc ${TD_OS_DIR}/inc) ADD_EXECUTABLE(taosdump ${SRC}) TARGET_LINK_LIBRARIES(taosdump taos_static) ENDIF () diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index a7874b7e5a..f168a0d90f 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "taos.h" #include "taosmsg.h" @@ -35,6 +36,11 @@ #define COMMAND_SIZE 65536 #define DEFAULT_DUMP_FILE "taosdump.sql" +int converStringToReadable(char *str, int size, char *buf, int bufsize); +int convertNCharToReadable(char *str, int size, char *buf, int bufsize); +void taosDumpCharset(FILE *fp); +void taosLoadFileCharset(FILE *fp, char *fcharset); + typedef struct { short bytes; int8_t type; @@ -60,8 +66,7 @@ enum _show_db_index { TSDB_MAX_SHOW_DB }; -// -----------------------------------------SHOW TABLES CONFIGURE -// ------------------------------------- +// -----------------------------------------SHOW TABLES CONFIGURE ------------------------------------- enum _show_tables_index { TSDB_SHOW_TABLES_NAME_INDEX, TSDB_SHOW_TABLES_CREATED_TIME_INDEX, @@ -70,8 +75,7 @@ enum _show_tables_index { TSDB_MAX_SHOW_TABLES }; -// ---------------------------------- DESCRIBE METRIC CONFIGURE -// ------------------------------ +// ---------------------------------- DESCRIBE METRIC CONFIGURE ------------------------------ enum _describe_table_index { TSDB_DESCRIBE_METRIC_FIELD_INDEX, TSDB_DESCRIBE_METRIC_TYPE_INDEX, @@ -84,7 +88,7 @@ typedef struct { char field[TSDB_COL_NAME_LEN + 1]; char type[16]; int length; - char note[8]; + char note[128]; } SColDes; typedef struct { @@ -149,6 +153,7 @@ static struct argp_option options[] = { {"output", 'o', "OUTPUT", 0, "Output file name.", 1}, {"input", 'i', "INPUT", 0, "Input file name.", 1}, {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1}, + {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1}, // dump unit options {"all-databases", 'A', 0, 0, "Dump all databases.", 2}, {"databases", 'B', 0, 0, "Dump assigned databases", 2}, @@ -171,6 +176,7 @@ struct arguments { // output file char output[TSDB_FILENAME_LEN + 1]; char input[TSDB_FILENAME_LEN + 1]; + char *encode; // dump unit option bool all_databases; bool databases; @@ -212,7 +218,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'P': arguments->port = atoi(arg); break; - // output file + // output file case 'o': if (wordexp(arg, &full_path, 0) != 0) { fprintf(stderr, "Invalid path %s\n", arg); @@ -238,14 +244,17 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { strcpy(configDir, full_path.we_wordv[0]); wordfree(&full_path); break; - // dump unit option + case 'e': + arguments->encode = arg; + break; + // dump unit option case 'A': arguments->all_databases = true; break; case 'B': arguments->databases = true; break; - // dump format option + // dump format option case 's': arguments->schemaonly = true; break; @@ -283,6 +292,7 @@ static struct argp argp = {options, parse_opt, args_doc, doc}; TAOS *taos = NULL; TAOS_RES *result = NULL; char *command = NULL; +char *lcommand = NULL; char *buffer = NULL; int taosDumpOut(struct arguments *arguments); @@ -296,7 +306,7 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp); void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, struct arguments *arguments, FILE *fp); void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, struct arguments *arguments, - FILE *fp); + FILE *fp); int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp); @@ -313,7 +323,7 @@ int main(int argc, char *argv[]) { // connection option NULL, "root", "taosdata", 0, // output file - DEFAULT_DUMP_FILE, DEFAULT_DUMP_FILE, + DEFAULT_DUMP_FILE, DEFAULT_DUMP_FILE, NULL, // dump unit option false, false, // dump format option @@ -440,15 +450,15 @@ int taosDumpOut(struct arguments *arguments) { /* Connect to server */ taos = taos_connect(arguments->host, arguments->user, arguments->password, NULL, arguments->port); if (taos == NULL) { - fprintf(stderr, "failed to connect to TDEngine server\n"); + fprintf(stderr, "failed to connect to TDengine server\n"); goto _exit_failure; } - /* --------------------------------- Main Code - * -------------------------------- */ - /* if (arguments->databases || arguments->all_databases) { // dump part of - * databases or all databases */ + /* --------------------------------- Main Code -------------------------------- */ + /* if (arguments->databases || arguments->all_databases) { // dump part of databases or all databases */ /* */ + taosDumpCharset(fp); + sprintf(command, "show databases"); if (taos_query(taos, command) != 0) { fprintf(stderr, "failed to run command: %s, reason: %s\n", command, taos_errstr(taos)); @@ -483,7 +493,7 @@ int taosDumpOut(struct arguments *arguments) { continue; } - _dump_db_point: + _dump_db_point: dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo)); if (dbInfos[count] == NULL) { @@ -492,17 +502,19 @@ int taosDumpOut(struct arguments *arguments) { } strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes); - dbInfos[count]->replica = (int)(*((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX])); - dbInfos[count]->days = (int)(*((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX])); - dbInfos[count]->keep = *((int *)row[TSDB_SHOW_DB_KEEP_INDEX]); - dbInfos[count]->tables = *((int *)row[TSDB_SHOW_DB_TABLES_INDEX]); - dbInfos[count]->rows = *((int *)row[TSDB_SHOW_DB_ROWS_INDEX]); - dbInfos[count]->cache = *((int *)row[TSDB_SHOW_DB_CACHE_INDEX]); - dbInfos[count]->ablocks = *((int *)row[TSDB_SHOW_DB_ABLOCKS_INDEX]); - dbInfos[count]->tblocks = (int)(*((int16_t *)row[TSDB_SHOW_DB_TBLOCKS_INDEX])); - dbInfos[count]->ctime = *((int *)row[TSDB_SHOW_DB_CTIME_INDEX]); - dbInfos[count]->clog = (int)(*((int8_t *)row[TSDB_SHOW_DB_CLOG_INDEX])); - dbInfos[count]->comp = (int)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); + if (strcmp(arguments->user, "root") == 0) { + dbInfos[count]->replica = (int)(*((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX])); + dbInfos[count]->days = (int)(*((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX])); + dbInfos[count]->keep = *((int *)row[TSDB_SHOW_DB_KEEP_INDEX]); + dbInfos[count]->tables = *((int *)row[TSDB_SHOW_DB_TABLES_INDEX]); + dbInfos[count]->rows = *((int *)row[TSDB_SHOW_DB_ROWS_INDEX]); + dbInfos[count]->cache = *((int *)row[TSDB_SHOW_DB_CACHE_INDEX]); + dbInfos[count]->ablocks = *((int *)row[TSDB_SHOW_DB_ABLOCKS_INDEX]); + dbInfos[count]->tblocks = (int)(*((int16_t *)row[TSDB_SHOW_DB_TBLOCKS_INDEX])); + dbInfos[count]->ctime = *((int *)row[TSDB_SHOW_DB_CTIME_INDEX]); + dbInfos[count]->clog = (int)(*((int8_t *)row[TSDB_SHOW_DB_CLOG_INDEX])); + dbInfos[count]->comp = (int)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); + } count++; @@ -562,7 +574,7 @@ int taosDumpOut(struct arguments *arguments) { taosFreeDbInfos(); return 0; - _exit_failure: +_exit_failure: fclose(fp); taos_close(taos); taos_free_result(result); @@ -577,9 +589,9 @@ void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) { pstr += sprintf(pstr, "CREATE DATABASE IF NOT EXISTS %s", dbInfo->name); if (isDumpProperty) { pstr += sprintf(pstr, - " REPLICA %d DAYS %d KEEP %d TABLES %d ROWS %d CACHE %d ABLOCKS %d TBLOCKS %d CTIME %d CLOG %d COMP %d", - dbInfo->replica, dbInfo->days, dbInfo->keep, dbInfo->tables, dbInfo->rows, dbInfo->cache, - dbInfo->ablocks, dbInfo->tblocks, dbInfo->ctime, dbInfo->clog, dbInfo->comp); + " REPLICA %d DAYS %d KEEP %d TABLES %d ROWS %d CACHE %d ABLOCKS %d TBLOCKS %d CTIME %d CLOG %d COMP %d", + dbInfo->replica, dbInfo->days, dbInfo->keep, dbInfo->tables, dbInfo->rows, dbInfo->cache, + dbInfo->ablocks, dbInfo->tblocks, dbInfo->ctime, dbInfo->clog, dbInfo->comp); } fprintf(fp, "%s\n\n", buffer); @@ -614,7 +626,7 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp) { TAOS_FIELD *fields = taos_fetch_fields(result); - fd = open(".table.tmp", O_RDWR | O_CREAT, 0755); + fd = open(".table.tmp", O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); if (fd == -1) { fprintf(stderr, "failed to open temp file\n"); taos_free_result(result); @@ -720,8 +732,8 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols /* pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); */ } - /* if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || - * strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { */ + /* if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || strcasecmp(tableDes->cols[counter].type, "nchar") + * == 0) { */ /* pstr += sprintf(pstr, "(%d)", tableDes->cols[counter].length); */ /* } */ } @@ -819,7 +831,7 @@ int32_t taosDumpMetric(char *metric, struct arguments *arguments, FILE *fp) { return -1; } - fd = open(".table.tmp", O_RDWR | O_CREAT, 0755); + fd = open(".table.tmp", O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); if (fd < 0) { fprintf(stderr, "failed to open temp file"); return -1; @@ -855,7 +867,7 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments) { char *pstr = NULL; TAOS_ROW row = NULL; int numFields = 0; - int lstr = 0; + char *tbuf = NULL; if (arguments->schemaonly) return 0; @@ -875,6 +887,11 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments) { numFields = taos_field_count(taos); assert(numFields > 0); TAOS_FIELD *fields = taos_fetch_fields(result); + tbuf = (char *)malloc(COMMAND_SIZE); + if (tbuf == NULL) { + fprintf(stderr, "No enough memory\n"); + return -1; + } count = 0; while ((row = taos_fetch_row(result)) != NULL) { @@ -918,14 +935,13 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments) { break; case TSDB_DATA_TYPE_BINARY: *(pstr++) = '\''; - for (lstr = 0; lstr < fields[col].bytes; lstr++) { - if (((char *)row[col])[lstr] == '\0') break; - *(pstr++) = ((char *)row[col])[lstr]; - } + converStringToReadable((char *)row[col], fields[col].bytes, tbuf, COMMAND_SIZE); + pstr = stpcpy(pstr, tbuf); *(pstr++) = '\''; break; case TSDB_DATA_TYPE_NCHAR: - pstr += sprintf(pstr, "\'%s\'", (char *)row[col]); + convertNCharToReadable((char *)row[col], fields[col].bytes, tbuf, COMMAND_SIZE); + pstr += sprintf(pstr, "\'%s\'", tbuf); break; case TSDB_DATA_TYPE_TIMESTAMP: pstr += sprintf(pstr, "%ld", *(int64_t *)row[col]); @@ -949,6 +965,7 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments) { fprintf(fp, "\n"); + if (tbuf) free(tbuf); taos_free_result(result); result = NULL; return 0; @@ -966,7 +983,7 @@ int taosCheckParam(struct arguments *arguments) { } if (arguments->arg_list_len == 0) { if ((!arguments->all_databases) && (!arguments->isDumpIn)) { - fprintf(stderr, "taosdump requirs parameters\n"); + fprintf(stderr, "taosdump requires parameters\n"); return -1; } } @@ -976,6 +993,11 @@ int taosCheckParam(struct arguments *arguments) { return -1; } + if (!arguments->isDumpIn && arguments->encode != NULL) { + fprintf(stderr, "invalid option in dump out\n"); + return -1; + } + return 0; } @@ -990,15 +1012,64 @@ bool isEmptyCommand(char *cmd) { return true; } +void taosReplaceCtrlChar(char *str) { + _Bool ctrlOn = false; + char *pstr = NULL; + + for (pstr = str; *str != '\0'; ++str) { + if (ctrlOn) { + switch (*str) { + case 'n': + *pstr = '\n'; + pstr++; + break; + case 'r': + *pstr = '\r'; + pstr++; + break; + case 't': + *pstr = '\t'; + pstr++; + break; + case '\\': + *pstr = '\\'; + pstr++; + break; + case '\'': + *pstr = '\''; + pstr++; + break; + default: + break; + } + ctrlOn = false; + } else { + if (*str == '\\') { + ctrlOn = true; + } else { + *pstr = *str; + pstr++; + } + } + } + + *pstr = '\0'; +} + int taosDumpIn(struct arguments *arguments) { assert(arguments->isDumpIn); - int tsize = 0; - FILE *fp = NULL; - char *line = NULL; - bool isRun = true; - size_t line_size = 0; - char *pstr = NULL; + int tsize = 0; + FILE * fp = NULL; + char * line = NULL; + _Bool isRun = true; + size_t line_size = 0; + char * pstr = NULL, *lstr = NULL; + iconv_t cd = (iconv_t)-1; + size_t inbytesleft = 0; + size_t outbytesleft = COMMAND_SIZE; + char fcharset[64]; + char * tcommand = NULL; fp = fopen(arguments->input, "r"); if (fp == NULL) { @@ -1006,26 +1077,57 @@ int taosDumpIn(struct arguments *arguments) { return -1; } + taosLoadFileCharset(fp, fcharset); + taos = taos_connect(arguments->host, arguments->user, arguments->password, NULL, arguments->port); if (taos == NULL) { - fprintf(stderr, "failed to connect to TDEngine server\n"); + fprintf(stderr, "failed to connect to TDengine server\n"); goto _dumpin_exit_failure; } command = (char *)malloc(COMMAND_SIZE); - if (command == NULL) { + lcommand = (char *)malloc(COMMAND_SIZE); + if (command == NULL || lcommand == NULL) { fprintf(stderr, "failed to connect to allocate memory\n"); goto _dumpin_exit_failure; } + // Resolve locale + if (*fcharset != '\0') { + arguments->encode = fcharset; + } + + if (arguments->encode != NULL && strcasecmp(tsCharset, arguments->encode) != 0) { + cd = iconv_open(tsCharset, arguments->encode); + if (cd == (iconv_t)-1) { + fprintf(stderr, "Failed to open iconv handle\n"); + goto _dumpin_exit_failure; + } + } + pstr = command; + int64_t linenu = 0; while (1) { ssize_t size = getline(&line, &line_size, fp); + linenu++; if (size <= 0) break; if (size == 1) { if (pstr != command) { - if (taos_query(taos, command) != 0) - fprintf(stderr, "failed to run command %s reason:%s \ncontinue...\n", command, taos_errstr(taos)); + inbytesleft = pstr - command; + memset(lcommand, 0, COMMAND_SIZE); + pstr = command; + lstr = lcommand; + outbytesleft = COMMAND_SIZE; + if (cd != (iconv_t)-1) { + iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft); + tcommand = lcommand; + } else { + tcommand = command; + } + taosReplaceCtrlChar(tcommand); + if (taos_query(taos, tcommand) != 0) + fprintf(stderr, "linenu: %ld failed to run command %s reason:%s \ncontinue...\n", linenu, command, + taos_errstr(taos)); pstr = command; pstr[0] = '\0'; @@ -1059,8 +1161,21 @@ int taosDumpIn(struct arguments *arguments) { if (!isRun) continue; if (command != pstr && !isEmptyCommand(command)) { - if (taos_query(taos, command) != 0) - fprintf(stderr, "failed to run command %s reason: %s \ncontinue...\n", command, taos_errstr(taos)); + inbytesleft = pstr - command; + memset(lcommand, 0, COMMAND_SIZE); + pstr = command; + lstr = lcommand; + outbytesleft = COMMAND_SIZE; + if (cd != (iconv_t)-1) { + iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft); + tcommand = lcommand; + } else { + tcommand = command; + } + taosReplaceCtrlChar(tcommand); + if (taos_query(taos, tcommand) != 0) + fprintf(stderr, "linenu:%ld failed to run command %s reason: %s \ncontinue...\n", linenu, command, + taos_errstr(taos)); } pstr = command; @@ -1069,19 +1184,131 @@ int taosDumpIn(struct arguments *arguments) { } if (pstr != command) { - if (taos_query(taos, command) != 0) - fprintf(stderr, "failed to run command %s reason:%s \ncontinue...\n", command, taos_errstr(taos)); + inbytesleft = pstr - command; + memset(lcommand, 0, COMMAND_SIZE); + pstr = command; + lstr = lcommand; + outbytesleft = COMMAND_SIZE; + if (cd != (iconv_t)-1) { + iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft); + tcommand = lcommand; + } else { + tcommand = command; + } + taosReplaceCtrlChar(lcommand); + if (taos_query(taos, tcommand) != 0) + fprintf(stderr, "linenu:%ld failed to run command %s reason:%s \ncontinue...\n", linenu, command, + taos_errstr(taos)); } + if (cd != (iconv_t)-1) iconv_close(cd); tfree(line); tfree(command); + tfree(lcommand); taos_close(taos); fclose(fp); return 0; - _dumpin_exit_failure: +_dumpin_exit_failure: + if (cd != (iconv_t)-1) iconv_close(cd); tfree(command); + tfree(lcommand); taos_close(taos); fclose(fp); return -1; } + +char *ascii_literal_list[] = { + "\\x00", "\\x01", "\\x02", "\\x03", "\\x04", "\\x05", "\\x06", "\\x07", "\\x08", "\\t", "\\n", "\\x0b", "\\x0c", + "\\r", "\\x0e", "\\x0f", "\\x10", "\\x11", "\\x12", "\\x13", "\\x14", "\\x15", "\\x16", "\\x17", "\\x18", "\\x19", + "\\x1a", "\\x1b", "\\x1c", "\\x1d", "\\x1e", "\\x1f", " ", "!", "\\\"", "#", "$", "%", "&", + "\\'", "(", ")", "*", "+", ",", "-", ".", "/", "0", "1", "2", "3", + "4", "5", "6", "7", "8", "9", ":", ";", "<", "=", ">", "?", "@", + "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", + "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", + "[", "\\\\", "]", "^", "_", "`", "a", "b", "c", "d", "e", "f", "g", + "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", + "u", "v", "w", "x", "y", "z", "{", "|", "}", "~", "\\x7f", "\\x80", "\\x81", + "\\x82", "\\x83", "\\x84", "\\x85", "\\x86", "\\x87", "\\x88", "\\x89", "\\x8a", "\\x8b", "\\x8c", "\\x8d", "\\x8e", + "\\x8f", "\\x90", "\\x91", "\\x92", "\\x93", "\\x94", "\\x95", "\\x96", "\\x97", "\\x98", "\\x99", "\\x9a", "\\x9b", + "\\x9c", "\\x9d", "\\x9e", "\\x9f", "\\xa0", "\\xa1", "\\xa2", "\\xa3", "\\xa4", "\\xa5", "\\xa6", "\\xa7", "\\xa8", + "\\xa9", "\\xaa", "\\xab", "\\xac", "\\xad", "\\xae", "\\xaf", "\\xb0", "\\xb1", "\\xb2", "\\xb3", "\\xb4", "\\xb5", + "\\xb6", "\\xb7", "\\xb8", "\\xb9", "\\xba", "\\xbb", "\\xbc", "\\xbd", "\\xbe", "\\xbf", "\\xc0", "\\xc1", "\\xc2", + "\\xc3", "\\xc4", "\\xc5", "\\xc6", "\\xc7", "\\xc8", "\\xc9", "\\xca", "\\xcb", "\\xcc", "\\xcd", "\\xce", "\\xcf", + "\\xd0", "\\xd1", "\\xd2", "\\xd3", "\\xd4", "\\xd5", "\\xd6", "\\xd7", "\\xd8", "\\xd9", "\\xda", "\\xdb", "\\xdc", + "\\xdd", "\\xde", "\\xdf", "\\xe0", "\\xe1", "\\xe2", "\\xe3", "\\xe4", "\\xe5", "\\xe6", "\\xe7", "\\xe8", "\\xe9", + "\\xea", "\\xeb", "\\xec", "\\xed", "\\xee", "\\xef", "\\xf0", "\\xf1", "\\xf2", "\\xf3", "\\xf4", "\\xf5", "\\xf6", + "\\xf7", "\\xf8", "\\xf9", "\\xfa", "\\xfb", "\\xfc", "\\xfd", "\\xfe", "\\xff"}; + +int converStringToReadable(char *str, int size, char *buf, int bufsize) { + char *pstr = str; + char *pbuf = buf; + while (size > 0) { + if (*pstr == '\0') break; + pbuf = stpcpy(pbuf, ascii_literal_list[((uint8_t)(*pstr))]); + pstr++; + size--; + } + *pbuf = '\0'; + return 0; +} + +int convertNCharToReadable(char *str, int size, char *buf, int bufsize) { + char *pstr = str; + char *pbuf = buf; + // TODO + wchar_t wc; + while (size > 0) { + if (*pstr == '\0') break; + int byte_width = mbtowc(&wc, pstr, MB_CUR_MAX); + + if ((int)wc < 256) { + pbuf = stpcpy(pbuf, ascii_literal_list[(int)wc]); + } else { + memcpy(pbuf, pstr, byte_width); + pbuf += byte_width; + } + pstr += byte_width; + } + + *pbuf = '\0'; + + return 0; +} + +void taosDumpCharset(FILE *fp) { + char charsetline[256]; + + fseek(fp, 0, SEEK_SET); + sprintf(charsetline, "#!%s\n", tsCharset); + fwrite(charsetline, strlen(charsetline), 1, fp); +} + +void taosLoadFileCharset(FILE *fp, char *fcharset) { + char * line = NULL; + size_t line_size = 0; + + fseek(fp, 0, SEEK_SET); + ssize_t size = getline(&line, &line_size, fp); + if (size <= 2) { + goto _exit_no_charset; + } + + if (strncmp(line, "#!", 2) != 0) { + goto _exit_no_charset; + } + if (line[size - 1] == '\n') { + line[size - 1] = '\0'; + size--; + } + strcpy(fcharset, line + 2); + + tfree(line); + return; + +_exit_no_charset: + fseek(fp, 0, SEEK_SET); + *fcharset = '\0'; + tfree(line); + return; +} \ No newline at end of file diff --git a/src/modules/CMakeLists.txt b/src/modules/CMakeLists.txt index 28c2b3c1c1..562de72410 100644 --- a/src/modules/CMakeLists.txt +++ b/src/modules/CMakeLists.txt @@ -1,5 +1,4 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) ADD_SUBDIRECTORY(monitor) diff --git a/src/modules/http/CMakeLists.txt b/src/modules/http/CMakeLists.txt old mode 100755 new mode 100644 index 433931ac35..911c5a3515 --- a/src/modules/http/CMakeLists.txt +++ b/src/modules/http/CMakeLists.txt @@ -1,10 +1,17 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -IF (TD_LINUX) +IF (TD_LINUX_64) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc) + INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) + INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/src/inc ${TD_ROOT_DIR}/src/client/inc ${TD_OS_DIR}/inc ${TD_ROOT_DIR}/deps/zlib-1.2.11/inc inc) ADD_LIBRARY(http ${SRC}) TARGET_LINK_LIBRARIES(http taos_static z) + + IF (TD_CLUSTER) + TARGET_LINK_LIBRARIES(http http_admin) + ENDIF () ENDIF () diff --git a/src/modules/http/inc/httpHandle.h b/src/modules/http/inc/httpHandle.h index ae281612ec..e7ac0365c2 100644 --- a/src/modules/http/inc/httpHandle.h +++ b/src/modules/http/inc/httpHandle.h @@ -56,25 +56,25 @@ #define HTTP_REQTYPE_SINGLE_SQL 3 #define HTTP_REQTYPE_MULTI_SQL 4 -#define HTTP_CLOSE_CONN 0 -#define HTTP_KEEP_CONN 1 - -#define HTTP_PROCESS_ERROR 0 -#define HTTP_PROCESS_SUCCESS 1 - #define HTTP_CHECK_BODY_ERROR -1 #define HTTP_CHECK_BODY_CONTINUE 0 #define HTTP_CHECK_BODY_SUCCESS 1 -#define HTTP_READ_RETRY_TIMES 5 -#define HTTP_READ_WAIT_TIME_MS 5 #define HTTP_WRITE_RETRY_TIMES 500 #define HTTP_WRITE_WAIT_TIME_MS 5 #define HTTP_EXPIRED_TIME 60000 +#define HTTP_DELAY_CLOSE_TIME_MS 1000 #define HTTP_COMPRESS_IDENTITY 0 #define HTTP_COMPRESS_GZIP 2 +typedef enum { + HTTP_CONTEXT_STATE_READY, + HTTP_CONTEXT_STATE_HANDLING, + HTTP_CONTEXT_STATE_DROPPING, + HTTP_CONTEXT_STATE_CLOSED +} HttpContextState; + struct HttpContext; struct HttpThread; @@ -174,10 +174,9 @@ typedef struct HttpContext { uint8_t fromMemPool; uint8_t acceptEncoding; uint8_t contentEncoding; - uint8_t usedByEpoll; - uint8_t usedByApp; uint8_t reqType; uint8_t parsed; + int32_t state; char ipstr[22]; char user[TSDB_USER_LEN]; // parsed from auth token or login message char pass[TSDB_PASSWORD_LEN]; @@ -189,7 +188,7 @@ typedef struct HttpContext { HttpSqlCmds *multiCmds; JsonBuf *jsonBuf; HttpParser parser; - void *readTimer; + void *timer; struct HttpThread *pThread; struct HttpContext *prev; struct HttpContext *next; @@ -271,6 +270,7 @@ bool httpProcessData(HttpContext *pContext); bool httpReadDataImp(HttpContext *pContext); bool httpParseRequest(HttpContext* pContext); int httpCheckReadCompleted(HttpContext* pContext); +void httpReadDirtyData(HttpContext *pContext); // http request handler void httpProcessRequest(HttpContext *pContext); @@ -306,5 +306,9 @@ int httpGzipCompress(HttpContext *pContext, char *inSrcData, int32_t inSrcDataLe extern const char *httpKeepAliveStr[]; extern const char *httpVersionStr[]; +const char* httpContextStateStr(HttpContextState state); + +bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState); +void httpRemoveContextFromEpoll(HttpThread *pThread, HttpContext *pContext); #endif diff --git a/src/modules/http/src/cJSON.c b/src/modules/http/src/cJSON.c index 1649161a25..fa836f9871 100644 --- a/src/modules/http/src/cJSON.c +++ b/src/modules/http/src/cJSON.c @@ -529,7 +529,7 @@ static unsigned parse_hex4(const unsigned char * const input) * A literal can be one or two sequences of the form \uXXXX */ static unsigned char utf16_literal_to_utf8(const unsigned char * const input_pointer, const unsigned char * const input_end, unsigned char **output_pointer) { - long unsigned int codepoint = 0; + uint64_t codepoint = 0; unsigned int first_code = 0; const unsigned char *first_sequence = input_pointer; unsigned char utf8_length = 0; diff --git a/src/modules/http/src/httpHandle.c b/src/modules/http/src/httpHandle.c index 39f78124a6..16e8378fb8 100644 --- a/src/modules/http/src/httpHandle.c +++ b/src/modules/http/src/httpHandle.c @@ -391,20 +391,28 @@ bool httpDecodeRequest(HttpContext* pContext) { * Process the request from http pServer */ bool httpProcessData(HttpContext* pContext) { - pContext->usedByApp = 1; + + if (!httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_HANDLING)) { + httpTrace("context:%p, fd:%d, ip:%s, state:%s not in ready state, stop process request", + pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state)); + httpCloseContextByApp(pContext); + return false; + } // handle Cross-domain request if (strcmp(pContext->parser.method.pos, "OPTIONS") == 0) { httpTrace("context:%p, fd:%d, ip:%s, process options request", pContext, pContext->fd, pContext->ipstr); httpSendOptionResp(pContext, "process options request success"); - return HTTP_PROCESS_SUCCESS; - } - - if (!httpDecodeRequest(pContext)) { - httpCloseContextByApp(pContext); - return HTTP_PROCESS_SUCCESS; + } else { + if (!httpDecodeRequest(pContext)) { + /* + * httpCloseContextByApp has been called when parsing the error + */ + //httpCloseContextByApp(pContext); + } else { + httpProcessRequest(pContext); + } } - httpProcessRequest(pContext); - return HTTP_PROCESS_SUCCESS; + return true; } diff --git a/src/modules/http/src/httpJson.c b/src/modules/http/src/httpJson.c index 26e539bfdb..2bb768e801 100644 --- a/src/modules/http/src/httpJson.c +++ b/src/modules/http/src/httpJson.c @@ -450,4 +450,4 @@ void httpJsonPairStatus(JsonBuf* buf, int code) { httpJsonPair(buf, "desc", 4, tsError[code], (int)strlen(tsError[code])); } } -} +} \ No newline at end of file diff --git a/src/modules/http/src/httpResp.c b/src/modules/http/src/httpResp.c index 3bea8d4ac3..ce339df2bd 100644 --- a/src/modules/http/src/httpResp.c +++ b/src/modules/http/src/httpResp.c @@ -44,7 +44,7 @@ const char *httpRespTemplate[] = { "%s 200 OK\r\nAccess-Control-Allow-Origin:*\r\n%sAccess-Control-Allow-Methods:POST, GET, OPTIONS, DELETE, PUT\r\nAccess-Control-Allow-Headers:Accept, Content-Type\r\nContent-Type: application/json;charset=utf-8\r\nContent-Length: %d\r\n\r\n" }; -void httpSendErrResp(HttpContext *pContext, int httpCode, char *httpCodeStr, int errNo, char *desc) { +void httpSendErrorRespImp(HttpContext *pContext, int httpCode, char *httpCodeStr, int errNo, char *desc) { httpError("context:%p, fd:%d, ip:%s, code:%d, error:%d:%s", pContext, pContext->fd, pContext->ipstr, httpCode, errNo, desc); @@ -174,9 +174,9 @@ void httpSendErrorRespWithDesc(HttpContext *pContext, int errNo, char *desc) { } if (desc == NULL) { - httpSendErrResp(pContext, httpCode, httpCodeStr, errNo + 1000, httpMsg[errNo]); + httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo + 1000, httpMsg[errNo]); } else { - httpSendErrResp(pContext, httpCode, httpCodeStr, errNo + 1000, desc); + httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo + 1000, desc); } } @@ -184,7 +184,7 @@ void httpSendErrorResp(HttpContext *pContext, int errNo) { httpSendErrorRespWith void httpSendTaosdErrorResp(HttpContext *pContext, int errCode) { int httpCode = 400; - httpSendErrResp(pContext, httpCode, "Bad Request", errCode, tsError[errCode]); + httpSendErrorRespImp(pContext, httpCode, "Bad Request", errCode, tsError[errCode]); } void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg) { @@ -200,7 +200,7 @@ void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg) { } else {} } - httpSendErrResp(pContext, httpCode, "Bad Request", TSDB_CODE_INVALID_SQL, temp); + httpSendErrorRespImp(pContext, httpCode, "Bad Request", TSDB_CODE_INVALID_SQL, temp); } void httpSendSuccResp(HttpContext *pContext, char *desc) { diff --git a/src/modules/http/src/httpServer.c b/src/modules/http/src/httpServer.c index acbc47dab2..8b981e0f84 100644 --- a/src/modules/http/src/httpServer.c +++ b/src/modules/http/src/httpServer.c @@ -48,6 +48,33 @@ #define EPOLLWAKEUP (1u << 29) #endif +const char* httpContextStateStr(HttpContextState state) { + switch (state) { + case HTTP_CONTEXT_STATE_READY: + return "ready"; + case HTTP_CONTEXT_STATE_HANDLING: + return "handling"; + case HTTP_CONTEXT_STATE_DROPPING: + return "dropping"; + case HTTP_CONTEXT_STATE_CLOSED: + return "closed"; + default: + return "unknown"; + } +} + +void httpRemoveContextFromEpoll(HttpThread *pThread, HttpContext *pContext) { + if (pContext->fd >= 0) { + epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL); + taosCloseSocket(pContext->fd); + pContext->fd = -1; + } +} + +bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState) { + return (__sync_val_compare_and_swap_32(&pContext->state, srcState, destState) == srcState); +} + void httpFreeContext(HttpServer *pServer, HttpContext *pContext); /** @@ -73,6 +100,7 @@ HttpContext *httpCreateContext(HttpServer *pServer) { pContext->signature = pContext; pContext->httpVersion = HTTP_VERSION_10; pContext->lastAccessTime = taosGetTimestampSec(); + pContext->state = HTTP_CONTEXT_STATE_READY; return pContext; } @@ -87,34 +115,32 @@ void httpFreeContext(HttpServer *pServer, HttpContext *pContext) { } void httpCleanUpContextTimer(HttpContext *pContext) { - if (pContext->readTimer != NULL) { - taosTmrStopA(&pContext->readTimer); - pContext->readTimer = NULL; - httpTrace("context:%p, fd:%d, ip:%s, close readTimer:%p", pContext, pContext->fd, pContext->ipstr, pContext->readTimer); + if (pContext->timer != NULL) { + taosTmrStopA(&pContext->timer); + httpTrace("context:%p, ip:%s, close timer:%p", pContext, pContext->ipstr, pContext->timer); + pContext->timer = NULL; } } -void httpCleanUpContext(HttpThread *pThread, HttpContext *pContext) { - void *sigature = __sync_val_compare_and_swap_64(&pContext->signature, pContext->signature, 0); - if (sigature == NULL) { +void httpCleanUpContext(HttpContext *pContext) { + httpTrace("context:%p, start the clean up operation", pContext); + __sync_val_compare_and_swap_64(&pContext->signature, pContext, 0); + if (pContext->signature != NULL) { httpTrace("context:%p is freed by another thread.", pContext); return; } + HttpThread *pThread = pContext->pThread; + httpCleanUpContextTimer(pContext); - if (pContext->fd >= 0) { - epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL); - taosCloseSocket(pContext->fd); - pContext->fd = -1; - } + httpRemoveContextFromEpoll(pThread, pContext); httpRestoreSession(pContext); pthread_mutex_lock(&pThread->threadMutex); pThread->numOfFds--; - httpTrace("context:%p, ip:%s, thread:%s, numOfFds:%d, fd is cleaned up", pContext, pContext->ipstr, pThread->label, pThread->numOfFds); if (pThread->numOfFds < 0) { httpError("context:%p, ip:%s, thread:%s, number of FDs:%d shall never be negative", pContext, pContext->ipstr, pThread->label, pThread->numOfFds); @@ -142,6 +168,7 @@ void httpCleanUpContext(HttpThread *pThread, HttpContext *pContext) { pContext->pThread = 0; pContext->prev = 0; pContext->next = 0; + pContext->state = HTTP_CONTEXT_STATE_READY; // avoid double free httpFreeJsonBuf(pContext); @@ -157,11 +184,9 @@ bool httpInitContext(HttpContext *pContext) { pContext->httpChunked = HTTP_UNCUNKED; pContext->acceptEncoding = HTTP_COMPRESS_IDENTITY; pContext->contentEncoding = HTTP_COMPRESS_IDENTITY; - pContext->usedByEpoll = 1; - pContext->usedByApp = 0; pContext->reqType = HTTP_REQTYPE_OTHERS; pContext->encodeMethod = NULL; - pContext->readTimer = NULL; + pContext->timer = NULL; memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd)); HttpParser *pParser = &pContext->parser; @@ -173,55 +198,79 @@ bool httpInitContext(HttpContext *pContext) { return true; } + +void httpCloseContext(HttpThread *pThread, HttpContext *pContext) { + taosTmrReset(httpCleanUpContext, HTTP_DELAY_CLOSE_TIME_MS, pContext, pThread->pServer->timerHandle, &pContext->timer); + httpTrace("context:%p, fd:%d, ip:%s, state:%s will be closed after:%d ms, timer:%p", + pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), HTTP_DELAY_CLOSE_TIME_MS, pContext->timer); +} + void httpCloseContextByApp(HttpContext *pContext) { HttpThread *pThread = pContext->pThread; - if (pContext->signature != pContext || pContext->pThread != pThread) { - return; - } - pContext->parsed = false; - httpTrace("context:%p, fd:%d, ip:%s, app use finished, usedByEpoll:%d, usedByApp:%d, httpVersion:1.%d, keepAlive:%d", - pContext, pContext->fd, pContext->ipstr, pContext->usedByEpoll, pContext->usedByApp, pContext->httpVersion, - pContext->httpKeepAlive); - if (!pContext->usedByEpoll) { - httpCleanUpContext(pThread, pContext); - } else { - if (pContext->httpVersion == HTTP_VERSION_10 && pContext->httpKeepAlive != HTTP_KEEPALIVE_ENABLE) { - httpCleanUpContext(pThread, pContext); - } else if (pContext->httpVersion != HTTP_VERSION_10 && pContext->httpKeepAlive == HTTP_KEEPALIVE_DISABLE) { - httpCleanUpContext(pThread, pContext); + bool keepAlive = true; + if (pContext->httpVersion == HTTP_VERSION_10 && pContext->httpKeepAlive != HTTP_KEEPALIVE_ENABLE) { + keepAlive = false; + } else if (pContext->httpVersion != HTTP_VERSION_10 && pContext->httpKeepAlive == HTTP_KEEPALIVE_DISABLE) { + keepAlive = false; + } else {} + + if (keepAlive) { + if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_READY)) { + httpTrace("context:%p, fd:%d, ip:%s, last state:handling, keepAlive:true, reuse connect", + pContext, pContext->fd, pContext->ipstr); + } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_CLOSED)) { + httpRemoveContextFromEpoll(pThread, pContext); + httpTrace("context:%p, fd:%d, ip:%s, last state:dropping, keepAlive:true, close connect", + pContext, pContext->fd, pContext->ipstr); + httpCloseContext(pThread, pContext); + } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) { + httpTrace("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, reuse connect", + pContext, pContext->fd, pContext->ipstr); + } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) { + httpRemoveContextFromEpoll(pThread, pContext); + httpTrace("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, close connect", + pContext, pContext->fd, pContext->ipstr); + httpCloseContext(pThread, pContext); } else { - pContext->usedByApp = 0; + httpRemoveContextFromEpoll(pThread, pContext); + httpError("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:true, close connect", + pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state); + httpCloseContext(pThread, pContext); } + } else { + httpRemoveContextFromEpoll(pThread, pContext); + httpTrace("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:false, close connect", + pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state); + httpCloseContext(pThread, pContext); } } void httpCloseContextByServer(HttpThread *pThread, HttpContext *pContext) { - if (pContext->signature != pContext || pContext->pThread != pThread) { - return; - } - + httpRemoveContextFromEpoll(pThread, pContext); pContext->parsed = false; - httpTrace("context:%p, fd:%d, ip:%s, epoll use finished, usedByEpoll:%d, usedByApp:%d", - pContext, pContext->fd, pContext->ipstr, pContext->usedByEpoll, pContext->usedByApp); - - if (pContext->fd >= 0) { - epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL); - taosCloseSocket(pContext->fd); - pContext->fd = -1; - } - - if (!pContext->usedByApp) { - httpCleanUpContext(pThread, pContext); + + if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_DROPPING)) { + httpTrace("context:%p, fd:%d, ip:%s, epoll finished, still used by app", pContext, pContext->fd, pContext->ipstr); + } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_DROPPING)) { + httpTrace("context:%p, fd:%d, ip:%s, epoll already finished, wait app finished", pContext, pContext->fd, pContext->ipstr); + } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_CLOSED)) { + httpTrace("context:%p, fd:%d, ip:%s, epoll finished, close context", pContext, pContext->fd, pContext->ipstr); + httpCloseContext(pThread, pContext); + } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) { + httpTrace("context:%p, fd:%d, ip:%s, epoll finished, will be closed soon", pContext, pContext->fd, pContext->ipstr); + httpCloseContext(pThread, pContext); } else { - pContext->usedByEpoll = 0; + httpError("context:%p, fd:%d, ip:%s, unknown state:%d", pContext, pContext->fd, pContext->ipstr, pContext->state); + httpCloseContext(pThread, pContext); } } -void httpCloseContextByServerFromTimer(void *param, void *tmrId) { +void httpCloseContextByServerForExpired(void *param, void *tmrId) { HttpContext *pContext = (HttpContext *)param; - httpError("context:%p, fd:%d, ip:%s, read http body error, time expired, readTimer:%p", pContext, pContext->fd, pContext->ipstr, tmrId); + httpRemoveContextFromEpoll(pContext->pThread, pContext); + httpError("context:%p, fd:%d, ip:%s, read http body error, time expired, timer:%p", pContext, pContext->fd, pContext->ipstr, tmrId); httpSendErrorResp(pContext, HTTP_PARSE_BODY_ERROR); httpCloseContextByServer(pContext->pThread, pContext); } @@ -240,7 +289,7 @@ void httpCleanUpConnect(HttpServer *pServer) { taosCloseSocket(pThread->pollFd); while (pThread->pHead) { - httpCleanUpContext(pThread, pThread->pHead); + httpCleanUpContext(pThread->pHead); } pthread_cancel(pThread->thread); @@ -254,7 +303,8 @@ void httpCleanUpConnect(HttpServer *pServer) { } // read all the data, then just discard it -void httpReadDirtyData(int fd) { +void httpReadDirtyData(HttpContext *pContext) { + int fd = pContext->fd; char data[1024] = {0}; int len = (int)taosReadSocket(fd, data, 1024); while (len >= sizeof(data)) { @@ -273,7 +323,7 @@ bool httpReadDataImp(HttpContext *pContext) { } else if (nread < 0) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { httpTrace("context:%p, fd:%d, ip:%s, read from socket error:%d, wait another event", - pContext, pContext->fd, pContext->ipstr, errno); + pContext, pContext->fd, pContext->ipstr, errno); break; } else { httpError("context:%p, fd:%d, ip:%s, read from socket error:%d, close connect", @@ -285,9 +335,10 @@ bool httpReadDataImp(HttpContext *pContext) { } if (pParser->bufsize >= (HTTP_BUFFER_SIZE - HTTP_STEP_SIZE)) { - httpReadDirtyData(pContext->fd); + httpReadDirtyData(pContext); httpError("context:%p, fd:%d, ip:%s, thread:%s, request big than:%d", pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, HTTP_BUFFER_SIZE); + httpRemoveContextFromEpoll(pContext->pThread, pContext); httpSendErrorResp(pContext, HTTP_REQUSET_TOO_BIG); return false; } @@ -347,8 +398,8 @@ bool httpReadData(HttpThread *pThread, HttpContext *pContext) { int ret = httpCheckReadCompleted(pContext); if (ret == HTTP_CHECK_BODY_CONTINUE) { - taosTmrReset(httpCloseContextByServerFromTimer, HTTP_EXPIRED_TIME, pContext, pThread->pServer->timerHandle, &pContext->readTimer); - httpTrace("context:%p, fd:%d, ip:%s, not finished yet, try another times, readTimer:%p", pContext, pContext->fd, pContext->ipstr, pContext->readTimer); + taosTmrReset(httpCloseContextByServerForExpired, HTTP_EXPIRED_TIME, pContext, pThread->pServer->timerHandle, &pContext->timer); + httpTrace("context:%p, fd:%d, ip:%s, not finished yet, try another times, timer:%p", pContext, pContext->fd, pContext->ipstr, pContext->timer); return false; } else if (ret == HTTP_CHECK_BODY_SUCCESS){ httpCleanUpContextTimer(pContext); @@ -367,7 +418,7 @@ bool httpReadData(HttpThread *pThread, HttpContext *pContext) { } void httpProcessHttpData(void *param) { - HttpThread * pThread = (HttpThread *)param; + HttpThread *pThread = (HttpThread *)param; HttpContext *pContext; int fdNum; @@ -395,54 +446,56 @@ void httpProcessHttpData(void *param) { } if (events[i].events & EPOLLPRI) { - httpTrace("context:%p, fd:%d, ip:%s, EPOLLPRI events occured, close connect", pContext, pContext->fd, - pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, state:%s, EPOLLPRI events occured, accessed:%d, close connect", + pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes); + httpRemoveContextFromEpoll(pThread, pContext); httpCloseContextByServer(pThread, pContext); continue; } if (events[i].events & EPOLLRDHUP) { - httpTrace("context:%p, fd:%d, ip:%s, EPOLLRDHUP events occured, close connect", - pContext, pContext->fd, pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, state:%s, EPOLLRDHUP events occured, accessed:%d, close connect", + pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes); + httpRemoveContextFromEpoll(pThread, pContext); httpCloseContextByServer(pThread, pContext); continue; } if (events[i].events & EPOLLERR) { - httpTrace("context:%p, fd:%d, ip:%s, EPOLLERR events occured, close connect", pContext, pContext->fd, - pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, state:%s, EPOLLERR events occured, accessed:%d, close connect", + pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes); + httpRemoveContextFromEpoll(pThread, pContext); httpCloseContextByServer(pThread, pContext); continue; } if (events[i].events & EPOLLHUP) { - httpTrace("context:%p, fd:%d, ip:%s, EPOLLHUP events occured, close connect", pContext, pContext->fd, - pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, state:%s, EPOLLHUP events occured, accessed:%d, close connect", + pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes); + httpRemoveContextFromEpoll(pThread, pContext); httpCloseContextByServer(pThread, pContext); continue; } - if (pContext->usedByApp) { - httpTrace("context:%p, fd:%d, ip:%s, still used by app, accessTimes:%d, try again", - pContext, pContext->fd, pContext->ipstr, pContext->accessTimes); - continue; - } - - if (!httpReadData(pThread, pContext)) { + if (!httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) { + httpTrace("context:%p, fd:%d, ip:%s, state:%s, not in ready state, ignore read events", + pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state)); continue; } if (!pContext->pThread->pServer->online) { - httpTrace("context:%p, fd:%d, ip:%s, server is not online", pContext, pContext->fd, pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, state:%s, server is not online, accessed:%d, close connect", + pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes); + httpRemoveContextFromEpoll(pThread, pContext); + httpReadDirtyData(pContext); httpSendErrorResp(pContext, HTTP_SERVER_OFFLINE); httpCloseContextByServer(pThread, pContext); continue; - } - - __sync_fetch_and_add(&pThread->pServer->requestNum, 1); - - if (!(*(pThread->processData))(pContext)) { - httpCloseContextByServer(pThread, pContext); + } else { + if (httpReadData(pThread, pContext)) { + (*(pThread->processData))(pContext); + __sync_fetch_and_add(&pThread->pServer->requestNum, 1); + } } } } @@ -456,6 +509,7 @@ void httpAcceptHttpConnection(void *arg) { HttpThread * pThread; HttpServer * pServer; HttpContext * pContext; + int totalFds; pServer = (HttpServer *)arg; @@ -484,6 +538,18 @@ void httpAcceptHttpConnection(void *arg) { continue; } + totalFds = 1; + for (int i = 0; i < pServer->numOfThreads; ++i) { + totalFds += pServer->pThreads[i].numOfFds; + } + + if (totalFds > tsHttpCacheSessions * 20) { + httpError("fd:%d, ip:%s:%u, totalFds:%d larger than httpCacheSessions:%d*20, refuse connection", + connFd, inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port), totalFds, tsHttpCacheSessions); + taosCloseSocket(connFd); + continue; + } + taosKeepTcpAlive(connFd); taosSetNonblocking(connFd, 1); @@ -498,8 +564,9 @@ void httpAcceptHttpConnection(void *arg) { continue; } - httpTrace("context:%p, fd:%d, ip:%s:%u, thread:%s, accept a new connection", pContext, connFd, - inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port), pThread->label); + httpTrace("context:%p, fd:%d, ip:%s:%u, thread:%s, numOfFds:%d, totalFds:%d, accept a new connection", + pContext, connFd, inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port), pThread->label, + pThread->numOfFds, totalFds); pContext->fd = connFd; sprintf(pContext->ipstr, "%s:%d", inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); @@ -532,10 +599,6 @@ void httpAcceptHttpConnection(void *arg) { pthread_mutex_unlock(&(pThread->threadMutex)); - httpTrace("context:%p, fd:%d, ip:%s:%u, thread:%s, numOfFds:%d, begin read request", - pContext, connFd, inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port), pThread->label, - pThread->numOfFds); - // pick up next thread for next connection threadId++; threadId = threadId % pServer->numOfThreads; diff --git a/src/modules/http/src/httpSql.c b/src/modules/http/src/httpSql.c index 518bf6ac39..9254658d58 100644 --- a/src/modules/http/src/httpSql.c +++ b/src/modules/http/src/httpSql.c @@ -28,6 +28,7 @@ void *taos_connect_a(char *ip, char *user, char *pass, char *db, int port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos); void httpProcessMultiSql(HttpContext *pContext); +void taosNotePrint(const char * const format, ...); void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) { HttpContext *pContext = (HttpContext *)param; @@ -164,6 +165,7 @@ void httpProcessMultiSql(HttpContext *pContext) { char *sql = httpGetCmdsString(pContext, cmd->sql); httpDump("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, start query, sql:%s", pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, sql); + taosNotePrint(sql); taos_query_a(pContext->session->taos, sql, httpProcessMultiSqlCallBack, (void *)pContext); } @@ -296,6 +298,7 @@ void httpProcessSingleSqlCmd(HttpContext *pContext) { httpDump("context:%p, fd:%d, ip:%s, user:%s, start query, sql:%s", pContext, pContext->fd, pContext->ipstr, pContext->user, sql); + taosNotePrint(sql); taos_query_a(pSession->taos, sql, httpProcessSingleSqlCallBack, (void *)pContext); } diff --git a/src/modules/http/src/httpSystem.c b/src/modules/http/src/httpSystem.c index 43923d985c..5c0d9a69d6 100644 --- a/src/modules/http/src/httpSystem.c +++ b/src/modules/http/src/httpSystem.c @@ -33,7 +33,16 @@ #include "restHandle.h" #include "tgHandle.h" +#ifdef CLUSTER + void adminInitHandle(HttpServer* pServer); + void opInitHandle(HttpServer* pServer); +#else + void adminInitHandle(HttpServer* pServer) {} + void opInitHandle(HttpServer* pServer) {} +#endif + static HttpServer *httpServer = NULL; +void taosInitNote(int numOfNoteLines, int maxNotes); int httpInitSystem() { taos_init(); @@ -51,9 +60,14 @@ int httpInitSystem() { pthread_mutex_init(&httpServer->serverMutex, NULL); + if (tsHttpEnableRecordSql != 0) { + taosInitNote(tsNumOfLogLines / 10, 1); + } restInitHandle(httpServer); + adminInitHandle(httpServer); gcInitHandle(httpServer); tgInitHandle(httpServer); + opInitHandle(httpServer); return 0; } @@ -75,7 +89,7 @@ int httpStartSystem() { } if (httpServer->timerHandle == NULL) { - httpServer->timerHandle = taosTmrInit(5, 1000, 60000, "http"); + httpServer->timerHandle = taosTmrInit(tsHttpCacheSessions * 20 + 100, 1000, 60000, "http"); } if (httpServer->timerHandle == NULL) { httpError("http init timer failed"); diff --git a/src/modules/http/src/tgHandle.c b/src/modules/http/src/tgHandle.c index 43d9fca013..ac17d6da09 100644 --- a/src/modules/http/src/tgHandle.c +++ b/src/modules/http/src/tgHandle.c @@ -215,7 +215,7 @@ ParseEnd: } } -int tgParseSchema(const char *content, char*fileName) { +int tgParseSchema(char *content, char*fileName) { cJSON *root = cJSON_Parse(content); if (root == NULL) { httpError("failed to parse telegraf schema file:%s, invalid json format, content:%s", fileName, content); diff --git a/src/modules/monitor/CMakeLists.txt b/src/modules/monitor/CMakeLists.txt old mode 100755 new mode 100644 index 94f2b95342..ff8ddb1c2d --- a/src/modules/monitor/CMakeLists.txt +++ b/src/modules/monitor/CMakeLists.txt @@ -1,10 +1,12 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -IF (TD_LINUX) +IF (TD_LINUX_64) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) + INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) + INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(./src SRC) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/src/inc ${TD_ROOT_DIR}/src/client/inc ${TD_OS_DIR}/inc inc) ADD_LIBRARY(monitor ${SRC}) TARGET_LINK_LIBRARIES(monitor taos_static) ENDIF () diff --git a/src/modules/monitor/src/monitorSystem.c b/src/modules/monitor/src/monitorSystem.c index 4571423725..013d86050e 100644 --- a/src/modules/monitor/src/monitorSystem.c +++ b/src/modules/monitor/src/monitorSystem.c @@ -34,7 +34,9 @@ typedef enum { MONITOR_CMD_CREATE_DB, MONITOR_CMD_CREATE_TB_LOG, MONITOR_CMD_CREATE_MT_DN, + MONITOR_CMD_CREATE_MT_ACCT, MONITOR_CMD_CREATE_TB_DN, + MONITOR_CMD_CREATE_TB_ACCT_ROOT, MONITOR_CMD_CREATE_TB_SLOWQUERY, MONITOR_CMD_MAX } MonitorCommand; @@ -68,6 +70,12 @@ void monitorInitDatabaseCb(void *param, TAOS_RES *result, int code); void monitorStartTimer(); void monitorSaveSystemInfo(); void monitorSaveLog(int level, const char *const format, ...); +void monitorSaveAcctLog(char *acctId, int64_t currentPointsPerSecond, int64_t maxPointsPerSecond, + int64_t totalTimeSeries, int64_t maxTimeSeries, int64_t totalStorage, int64_t maxStorage, + int64_t totalQueryTime, int64_t maxQueryTime, int64_t totalInbound, int64_t maxInbound, + int64_t totalOutbound, int64_t maxOutbound, int64_t totalDbs, int64_t maxDbs, + int64_t totalUsers, int64_t maxUsers, int64_t totalStreams, int64_t maxStreams, + int64_t totalConns, int64_t maxConns, int8_t accessState); void (*monitorCountReqFp)(SCountInfo *info) = NULL; void monitorExecuteSQL(char *sql); @@ -80,6 +88,9 @@ int monitorInitSystem() { monitor = (MonitorConn *)malloc(sizeof(MonitorConn)); memset(monitor, 0, sizeof(MonitorConn)); taosTmrReset(monitorCheckDiskUsage, CHECK_INTERVAL, NULL, tscTmr, &monitor->diskTimer); + + startMonitor = monitorStartSystem; + stopMonitor = monitorStopSystem; return 0; } @@ -99,7 +110,11 @@ void monitorInitConn(void *para, void *unused) { monitor->state = MONITOR_STATE_INITIALIZING; if (monitor->privateIpStr[0] == 0) { +#ifdef CLUSTER + strcpy(monitor->privateIpStr, tsPrivateIp); +#else strcpy(monitor->privateIpStr, tsInternalIp); +#endif for (int i = 0; i < TSDB_IPv4ADDR_LEN; ++i) { if (monitor->privateIpStr[i] == '.') { monitor->privateIpStr[i] = '_'; @@ -149,7 +164,30 @@ void dnodeBuildMonitorSql(char *sql, int cmd) { tsMonitorDbName, IP_LEN_STR + 1); } else if (cmd == MONITOR_CMD_CREATE_TB_DN) { snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn_%s using %s.dn tags('%s')", tsMonitorDbName, +#ifdef CLUSTER + monitor->privateIpStr, tsMonitorDbName, tsPrivateIp); +#else monitor->privateIpStr, tsMonitorDbName, tsInternalIp); +#endif + } else if (cmd == MONITOR_CMD_CREATE_MT_ACCT) { + snprintf(sql, SQL_LENGTH, + "create table if not exists %s.acct(ts timestamp " + ", currentPointsPerSecond bigint, maxPointsPerSecond bigint" + ", totalTimeSeries bigint, maxTimeSeries bigint" + ", totalStorage bigint, maxStorage bigint" + ", totalQueryTime bigint, maxQueryTime bigint" + ", totalInbound bigint, maxInbound bigint" + ", totalOutbound bigint, maxOutbound bigint" + ", totalDbs smallint, maxDbs smallint" + ", totalUsers smallint, maxUsers smallint" + ", totalStreams smallint, maxStreams smallint" + ", totalConns smallint, maxConns smallint" + ", accessState smallint" + ") tags (acctId binary(%d))", + tsMonitorDbName, TSDB_USER_LEN + 1); + } else if (cmd == MONITOR_CMD_CREATE_TB_ACCT_ROOT) { + snprintf(sql, SQL_LENGTH, "create table if not exists %s.acct_%s using %s.acct tags('%s')", tsMonitorDbName, "root", + tsMonitorDbName, "root"); } else if (cmd == MONITOR_CMD_CREATE_TB_SLOWQUERY) { snprintf(sql, SQL_LENGTH, "create table if not exists %s.slowquery(ts timestamp, username " @@ -183,7 +221,12 @@ void monitorInitDatabaseCb(void *param, TAOS_RES *result, int code) { if (monitor->cmdIndex == MONITOR_CMD_CREATE_TB_LOG) { taosLogFp = monitorSaveLog; taosLogSqlFp = monitorExecuteSQL; +#ifdef CLUSTER + taosLogAcctFp = monitorSaveAcctLog; + monitorLPrint("dnode:%s is started", tsPrivateIp); +#else monitorLPrint("dnode:%s is started", tsInternalIp); +#endif } monitor->cmdIndex++; monitorInitDatabase(); @@ -199,7 +242,11 @@ void monitorStopSystem() { return; } +#ifdef CLUSTER + monitorLPrint("dnode:%s is stopped", tsPrivateIp); +#else monitorLPrint("dnode:%s is stopped", tsInternalIp); +#endif monitor->state = MONITOR_STATE_STOPPED; taosLogFp = NULL; if (monitor->initTimer != NULL) { @@ -219,6 +266,16 @@ void monitorStartTimer() { taosTmrReset(monitorSaveSystemInfo, tsMonitorInterval * 1000, NULL, tscTmr, &monitor->timer); } +void dnodeMontiorInsertAcctCallback(void *param, TAOS_RES *result, int code) { + if (code < 0) { + monitorError("monitor:%p, save account info failed, code:%d", monitor->conn, code); + } else if (code == 0) { + monitorError("monitor:%p, save account info failed, affect rows:%d", monitor->conn, code); + } else { + monitorTrace("monitor:%p, save account info success, code:%d", monitor->conn, code); + } +} + void dnodeMontiorInsertSysCallback(void *param, TAOS_RES *result, int code) { if (code < 0) { monitorError("monitor:%p, save system info failed, code:%d %s", monitor->conn, code, monitor->sql); @@ -333,6 +390,35 @@ void monitorSaveSystemInfo() { } } +void monitorSaveAcctLog(char *acctId, int64_t currentPointsPerSecond, int64_t maxPointsPerSecond, + int64_t totalTimeSeries, int64_t maxTimeSeries, int64_t totalStorage, int64_t maxStorage, + int64_t totalQueryTime, int64_t maxQueryTime, int64_t totalInbound, int64_t maxInbound, + int64_t totalOutbound, int64_t maxOutbound, int64_t totalDbs, int64_t maxDbs, + int64_t totalUsers, int64_t maxUsers, int64_t totalStreams, int64_t maxStreams, + int64_t totalConns, int64_t maxConns, int8_t accessState) { + char sql[1024] = {0}; + sprintf(sql, + "insert into %s.acct_%s using %s.acct tags('%s') values(now" + ", %ld, %ld " + ", %ld, %ld" + ", %ld, %ld" + ", %ld, %ld" + ", %ld, %ld" + ", %ld, %ld" + ", %ld, %ld" + ", %ld, %ld" + ", %ld, %ld" + ", %ld, %ld" + ", %d)", + tsMonitorDbName, acctId, tsMonitorDbName, acctId, currentPointsPerSecond, maxPointsPerSecond, totalTimeSeries, + maxTimeSeries, totalStorage, maxStorage, totalQueryTime, maxQueryTime, totalInbound, maxInbound, + totalOutbound, maxOutbound, totalDbs, maxDbs, totalUsers, maxUsers, totalStreams, maxStreams, totalConns, + maxConns, accessState); + + monitorTrace("monitor:%p, save account info, sql %s", monitor->conn, sql); + taos_query_a(monitor->conn, sql, dnodeMontiorInsertAcctCallback, "account"); +} + void monitorSaveLog(int level, const char *const format, ...) { va_list argpointer; char sql[SQL_LENGTH] = {0}; @@ -350,7 +436,11 @@ void monitorSaveLog(int level, const char *const format, ...) { va_end(argpointer); if (len > max_length) len = max_length; +#ifdef CLUSTER + len += sprintf(sql + len, "', '%s')", tsPrivateIp); +#else len += sprintf(sql + len, "', '%s')", tsInternalIp); +#endif sql[len++] = 0; monitorTrace("monitor:%p, save log, sql: %s", monitor->conn, sql); diff --git a/src/os/CMakeLists.txt b/src/os/CMakeLists.txt index 9a4d557295..f88209ad9b 100644 --- a/src/os/CMakeLists.txt +++ b/src/os/CMakeLists.txt @@ -1,5 +1,4 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) ADD_SUBDIRECTORY(linux) diff --git a/src/os/darwin/CMakeLists.txt b/src/os/darwin/CMakeLists.txt index a10871c6f1..116d0ec8c0 100644 --- a/src/os/darwin/CMakeLists.txt +++ b/src/os/darwin/CMakeLists.txt @@ -1,9 +1,9 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -IF (TD_DARWIN) +IF (TD_DARWIN_64) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) + INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) - INCLUDE_DIRECTORIES(inc ${TD_ROOT_DIR}/src/inc) ADD_LIBRARY(os ${SRC}) ENDIF () diff --git a/src/os/darwin/inc/os.h b/src/os/darwin/inc/os.h index 217981d90c..c9f461d9a0 100644 --- a/src/os/darwin/inc/os.h +++ b/src/os/darwin/inc/os.h @@ -132,4 +132,9 @@ typedef int(*__compar_fn_t)(const void *, const void *); #define PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE #endif +#define BUILDIN_CLZL(val) __builtin_clzl(val) +#define BUILDIN_CLZ(val) __builtin_clz(val) +#define BUILDIN_CTZL(val) __builtin_ctzl(val) +#define BUILDIN_CTZ(val) __builtin_ctz(val) + #endif \ No newline at end of file diff --git a/src/os/linux/CMakeLists.txt b/src/os/linux/CMakeLists.txt index 2444d438de..8a4cc56f72 100644 --- a/src/os/linux/CMakeLists.txt +++ b/src/os/linux/CMakeLists.txt @@ -1,10 +1,10 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -IF (TD_LINUX) +IF (TD_LINUX_64) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) + INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) - INCLUDE_DIRECTORIES(inc ${TD_ROOT_DIR}/src/inc) ADD_LIBRARY(os ${SRC}) TARGET_LINK_LIBRARIES(os m rt) ENDIF () diff --git a/src/os/linux/inc/os.h b/src/os/linux/inc/os.h index 35ef811a8f..004896960e 100644 --- a/src/os/linux/inc/os.h +++ b/src/os/linux/inc/os.h @@ -13,40 +13,53 @@ * along with this program. If not, see . */ - #ifndef TDENGINE_PLATFORM_LINUX_H #define TDENGINE_PLATFORM_LINUX_H +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#include +#include #include +#include #include +#include +#include #include -#include -#include -#include -#include -#include -#include #include #include #include #include +#include +#include +#include +#include +#include +#include +#include #include #include #include #include #include +#include #include #include #include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #define taosCloseSocket(x) \ { \ @@ -91,7 +104,7 @@ void __sync_val_restore_32(int32_t *ptr, int32_t newval); (__a < __b) ? __a : __b; \ }) -#define MILLISECOND_PER_SECOND (1000L) +#define MILLISECOND_PER_SECOND ((int64_t)1000L) #define tsem_t sem_t #define tsem_init sem_init @@ -130,4 +143,8 @@ int64_t str2int64(char *str); #define BUILDIN_CTZL(val) __builtin_ctzl(val) #define BUILDIN_CTZ(val) __builtin_ctz(val) -#endif \ No newline at end of file +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/linux/src/tsystem.c b/src/os/linux/src/tsystem.c index 0989f007b3..fc2d9860d7 100644 --- a/src/os/linux/src/tsystem.c +++ b/src/os/linux/src/tsystem.c @@ -32,6 +32,7 @@ #include "tutil.h" #define PROCESS_ITEM 12 +extern char dataDir[TSDB_FILENAME_LEN]; typedef struct { uint64_t user; @@ -179,8 +180,7 @@ void taosGetSystemTimezone() { *lineEnd = 0; } - // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment - // variables + // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables if (strlen(buf) > 0) { setenv("TZ", buf, 1); } diff --git a/src/os/windows/CMakeLists.txt b/src/os/windows/CMakeLists.txt index 5e5bd93265..dc60b736ea 100644 --- a/src/os/windows/CMakeLists.txt +++ b/src/os/windows/CMakeLists.txt @@ -1,10 +1,11 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -IF (TD_WINDOWS) +IF (TD_WINDOWS_64) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) + INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) - INCLUDE_DIRECTORIES(inc ${TD_ROOT_DIR}/deps/pthread ${TD_ROOT_DIR}/src/inc) ADD_LIBRARY(os ${SRC}) TARGET_LINK_LIBRARIES(os winmm IPHLPAPI ws2_32) ENDIF () diff --git a/src/os/windows/inc/os.h b/src/os/windows/inc/os.h index 48e371376b..1a840ed1ef 100644 --- a/src/os/windows/inc/os.h +++ b/src/os/windows/inc/os.h @@ -25,6 +25,10 @@ #include #include "winsock2.h" #include +#include +#include +#include +#include #ifdef __cplusplus extern "C" { @@ -173,6 +177,10 @@ int fsendfile(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count); #define ssize_t int +#define strdup _strdup + +char *strndup(const char *s, size_t n); + #ifdef __cplusplus } #endif diff --git a/src/os/windows/src/twindows.c b/src/os/windows/src/twindows.c index 75bc8c2839..b1c3112bdb 100644 --- a/src/os/windows/src/twindows.c +++ b/src/os/windows/src/twindows.c @@ -31,6 +31,7 @@ char configDir[TSDB_FILENAME_LEN] = "C:/TDengine/cfg"; char tsDirectory[TSDB_FILENAME_LEN] = "C:/TDengine/data"; char logDir[TSDB_FILENAME_LEN] = "C:/TDengine/log"; +char dataDir[TSDB_FILENAME_LEN] = "C:/TDengine/data"; char scriptDir[TSDB_FILENAME_LEN] = "C:/TDengine/script"; bool taosCheckPthreadValid(pthread_t thread) { @@ -279,3 +280,14 @@ int32_t BUILDIN_CTZ(uint32_t val) { return (int)(r >> 3); } +char *strndup(const char *s, size_t n) { + int len = strlen(s); + if (len >= n) { + len = n; + } + + char *r = calloc(len + 1, 1); + memcpy(r, s, len); + r[len] = 0; + return r; +} \ No newline at end of file diff --git a/src/os/windows/src/twintcpclient.c b/src/os/windows/src/twintcpclient.c index 68a4902327..17293c3915 100644 --- a/src/os/windows/src/twintcpclient.c +++ b/src/os/windows/src/twintcpclient.c @@ -16,24 +16,24 @@ #include "tlog.h" void *taosInitTcpClient(char *ip, short port, char *label, int num, void *fp, void *shandle) { - tError("taosInitTcpClient not support in windows"); + tError("InitTcpClient not support in windows"); return 0; } void taosCloseTcpClientConnection(void *chandle) { - tError("taosCloseTcpClientConnection not support in windows"); + tError("CloseTcpClientConnection not support in windows"); } void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, short port) { - tError("taosOpenTcpClientConnection not support in windows"); + tError("OpenTcpClientConnection not support in windows"); return 0; } int taosSendTcpClientData(unsigned int ip, short port, char *data, int len, void *chandle) { - tError("taosSendTcpClientData not support in windows"); + tError("SendTcpClientData not support in windows"); return 0; } void taosCleanUpTcpClient(void *chandle) { - tError("taosSendTcpClientData not support in windows"); + tError("SendTcpClientData not support in windows"); } diff --git a/src/os/windows/src/twintcpserver.c b/src/os/windows/src/twintcpserver.c index 5d504f7432..a51d807aa2 100644 --- a/src/os/windows/src/twintcpserver.c +++ b/src/os/windows/src/twintcpserver.c @@ -16,19 +16,19 @@ #include "tlog.h" void taosCloseTcpServerConnection(void *chandle) { - tError("taosCloseTcpServerConnection not support in windows"); + tError("CloseTcpServerConnection not support in windows"); } void taosCleanUpTcpServer(void *handle) { - tError("taosCleanUpTcpServer not support in windows"); + tError("CleanUpTcpServer not support in windows"); } void *taosInitTcpServer(char *ip, short port, char *label, int numOfThreads, void *fp, void *shandle) { - tError("taosInitTcpServer not support in windows"); + tError("InitTcpServer not support in windows"); return 0; } int taosSendTcpServerData(unsigned int ip, short port, char *data, int len, void *chandle) { - tError("taosSendTcpServerData not support in windows"); + tError("SendTcpServerData not support in windows"); return 0; } diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt old mode 100755 new mode 100644 index a6a9fdd19b..b3a761e3df --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -1,18 +1,19 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -INCLUDE_DIRECTORIES(inc ${TD_ROOT_DIR}/src/inc ${TD_OS_DIR}/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) +INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) +INCLUDE_DIRECTORIES(inc) -IF (TD_LINUX) +IF (TD_LINUX_64) AUX_SOURCE_DIRECTORY(./src SRC) -ELSEIF (TD_DARWIN) +ELSEIF (TD_DARWIN_64) LIST(APPEND SRC ./src/thaship.c) LIST(APPEND SRC ./src/trpc.c) LIST(APPEND SRC ./src/tstring.c) LIST(APPEND SRC ./src/tudp.c) -ELSEIF (TD_WINDOWS) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/deps/pthread) +ELSEIF (TD_WINDOWS_64) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) LIST(APPEND SRC ./src/thaship.c) LIST(APPEND SRC ./src/trpc.c) LIST(APPEND SRC ./src/tstring.c) diff --git a/src/rpc/src/trpc.c b/src/rpc/src/trpc.c index a91ce7a213..6f92154e9e 100644 --- a/src/rpc/src/trpc.c +++ b/src/rpc/src/trpc.c @@ -74,7 +74,7 @@ typedef struct { void * chandle; // handle passed by TCP/UDP connection layer void * ahandle; // handle returned by upper app layter int retry; - int tretry; // total retry + int tretry; // total retry void * pTimer; void * pIdleTimer; char * pRspMsg; @@ -87,7 +87,7 @@ typedef struct { typedef struct { int sessions; - void * qhandle; // for scheduler + void * qhandle; // for scheduler SRpcConn * connList; void * idPool; void * tmrCtrl; @@ -102,11 +102,11 @@ typedef struct rpc_server { int mask; int numOfChanns; int numOfThreads; - int idMgmt; // ID management method + int idMgmt; // ID management method int type; - int idleTime; // milliseconds; - int noFree; // do not free the request msg when rsp is received - int index; // for UDP server, next thread for new connection + int idleTime; // milliseconds; + int noFree; // do not free the request msg when rsp is received + int index; // for UDP server, next thread for new connection short localPort; char label[12]; void *(*fp)(char *, void *ahandle, void *thandle); @@ -115,11 +115,8 @@ typedef struct rpc_server { SRpcChann *channList; } STaosRpc; -// configurable -uint32_t rpcDebugFlag = 131; -int tsRpcTimer = 300; -int tsRpcMaxTime = 600; // seconds; -int tsRpcProgressTime = 10; // milliseocnds + +int tsRpcProgressTime = 10; // milliseocnds // not configurable int tsRpcMaxRetry; @@ -135,19 +132,22 @@ int (*taosSendData[])(uint32_t ip, short port, char *data, int len, void *chandl taosSendUdpData, taosSendUdpData, taosSendTcpServerData, taosSendTcpClientData}; void *(*taosOpenConn[])(void *shandle, void *thandle, char *ip, short port) = { - taosOpenUdpConnection, taosOpenUdpConnection, NULL, taosOpenTcpClientConnection, + taosOpenUdpConnection, + taosOpenUdpConnection, + NULL, + taosOpenTcpClientConnection, }; void (*taosCloseConn[])(void *chandle) = {NULL, NULL, taosCloseTcpServerConnection, taosCloseTcpClientConnection}; -int taosReSendRspToPeer(SRpcConn *pConn); +int taosReSendRspToPeer(SRpcConn *pConn); void taosProcessTaosTimer(void *, void *); void *taosProcessDataFromPeer(char *data, int dataLen, uint32_t ip, short port, void *shandle, void *thandle, void *chandle); -int taosSendDataToPeer(SRpcConn *pConn, char *data, int dataLen); -void taosProcessSchedMsg(SSchedMsg *pMsg); -int taosAuthenticateMsg(uint8_t *pMsg, int msgLen, uint8_t *pAuth, uint8_t *pKey); -int taosBuildAuthHeader(uint8_t *pMsg, int msgLen, uint8_t *pAuth, uint8_t *pKey); +int taosSendDataToPeer(SRpcConn *pConn, char *data, int dataLen); +void taosProcessSchedMsg(SSchedMsg *pMsg); +int taosAuthenticateMsg(uint8_t *pMsg, int msgLen, uint8_t *pAuth, uint8_t *pKey); +int taosBuildAuthHeader(uint8_t *pMsg, int msgLen, uint8_t *pAuth, uint8_t *pKey); char *taosBuildReqHeader(void *param, char type, char *msg) { STaosHeader *pHeader; @@ -538,7 +538,7 @@ int taosGetRpcConn(int chann, int sid, char *meterId, STaosRpc *pServer, SRpcCon if (ret != 0) { tWarn("%s cid:%d sid:%d id:%s, meterId not there pConn:%p", pServer->label, chann, sid, pConn->meterId, pConn); taosFreeId(pChann->idPool, sid); // sid shall be released - memset(pConn, 0, sizeof(SRpcConn)); + memset(pConn, 0, sizeof(SRpcConn)); return ret; } } @@ -1096,7 +1096,7 @@ void *taosProcessDataFromPeer(char *data, int dataLen, uint32_t ip, short port, pHeader->msgLen = msgLen - (int)sizeof(STaosHeader) + (int)sizeof(SIntMsg); if (pHeader->spi) pHeader->msgLen -= sizeof(STaosDigest); - if ((pHeader->msgType & 1) == 0 && (pHeader->content[0] == TSDB_CODE_SESSION_ALREADY_EXIST)) { + if ((pHeader->msgType & 1) == 0 && (pHeader->content[0] == TSDB_CODE_INVALID_VALUE)) { schedMsg.msg = NULL; // connection shall be closed } else { schedMsg.msg = (char *)(&(pHeader->destId)); @@ -1275,7 +1275,7 @@ void taosProcessTaosTimer(void *param, void *tmrId) { pConn->pTimer = NULL; pConn->retry++; - if (pConn->retry < 3) { + if (pConn->retry < 4) { tTrace("%s cid:%d sid:%d id:%s, re-send msg:%s to %s:%hu pConn:%p", pServer->label, pConn->chann, pConn->sid, pConn->meterId, taosMsg[pConn->outType], pConn->peerIpstr, pConn->peerPort, pConn); if (pConn->pMsgNode && pConn->pMsgNode->msgLen > 0) { @@ -1308,7 +1308,7 @@ void taosProcessTaosTimer(void *param, void *tmrId) { if (pHeader) { (*taosSendData[pServer->type])(pConn->peerIp, pConn->peerPort, (char *)pHeader, msgLen, pConn->chandle); - taosTmrReset(taosProcessTaosTimer, tsRpcTimer, pConn, pChann->tmrCtrl, &pConn->pTimer); + taosTmrReset(taosProcessTaosTimer, tsRpcTimer<retry, pConn, pChann->tmrCtrl, &pConn->pTimer); } pthread_mutex_unlock(&pChann->mutex); diff --git a/src/rpc/src/tstring.c b/src/rpc/src/tstring.c index d62cbaf7d3..c7afe54d9a 100644 --- a/src/rpc/src/tstring.c +++ b/src/rpc/src/tstring.c @@ -179,7 +179,7 @@ char *tsError[] = {"success", "can't remove dnode which is master", "wrong schema", "no results", - "num of users execeed maxUsers", + "num of users execeed maxUsers", //55 "num of databases execeed maxDbs", "num of tables execeed maxTables", "num of dnodes execeed maxDnodes", @@ -208,32 +208,34 @@ char *tsError[] = {"success", "client out of memory", "data value overflow", "query cancelled", - "grant timeseries limited", // 84 + "grant timeseries limited", "grant expired", // 85 "client no disk space", "DB file corrupted", "version of client and server not match", "invalid account parameter", - "no enough available time series", + "no enough available time series", //90 "storage credit is used up", - "query credit is used up", // 92 + "query credit is used up", "grant database limited", "grant user limited", - "grant connection limited", + "grant connection limited", //95 "grant stream limited", "grant writing speed limited", "grant storage limited", - "grant query time limited", // 99 - "grant account limited", + "grant query time limited", + "grant account limited", // 100 "grant dnode limited", - "grant cpu core limited", // 102 + "grant cpu core limited", "session not ready", "batch size too big", - "timestamp out of range", + "timestamp out of range", //105 "invalid query message", "timestamp disordered in cache block", "timestamp disordered in file block", - "invalid commit log", - "server no disk space", + "invalid commit log", //110 + "server no disk space", + "only super table has metric meta info", + "tags value not unique for join", "invalid submit message", }; diff --git a/src/rpc/src/ttcpclient.c b/src/rpc/src/ttcpclient.c index db00b8d931..e12f1e1728 100644 --- a/src/rpc/src/ttcpclient.c +++ b/src/rpc/src/ttcpclient.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -26,16 +25,16 @@ #include "os.h" #include "taosmsg.h" #include "tlog.h" -#include "tlog.h" #include "tsocket.h" #include "ttcpclient.h" #include "tutil.h" #ifndef EPOLLWAKEUP - #define EPOLLWAKEUP (1u << 29) +#define EPOLLWAKEUP (1u << 29) #endif typedef struct _tcp_fd { + void *signature; int fd; // TCP socket FD void * thandle; uint32_t ip; @@ -66,6 +65,7 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { STcpClient *pTcp; if (pFdObj == NULL) return; + if (pFdObj->signature != pFdObj) return; pTcp = pFdObj->pTcp; if (pTcp == NULL) { @@ -155,23 +155,40 @@ static void *taosReadTcpData(void *param) { } void *buffer = malloc(1024); - int headLen = taosReadMsg(pFdObj->fd, buffer, sizeof(STaosHeader)); + if (NULL == buffer) { + tTrace("%s TCP malloc(size:1024) fail\n", pTcp->label); + taosCleanUpTcpFdObj(pFdObj); + continue; + } + + int headLen = taosReadMsg(pFdObj->fd, buffer, sizeof(STaosHeader)); if (headLen != sizeof(STaosHeader)) { tError("%s read error, headLen:%d", pTcp->label, headLen); + tfree(buffer); taosCleanUpTcpFdObj(pFdObj); continue; } int dataLen = (int32_t)htonl((uint32_t)((STaosHeader *)buffer)->msgLen); - if (dataLen > 1024) buffer = realloc(buffer, (size_t)dataLen); + if (dataLen > 1024) { + void *b = realloc(buffer, (size_t)dataLen); + if (NULL == b) { + tTrace("%s TCP malloc(size:%d) fail\n", pTcp->label, dataLen); + tfree(buffer); + taosCleanUpTcpFdObj(pFdObj); + continue; + } + buffer = b; + } int leftLen = dataLen - headLen; int retLen = taosReadMsg(pFdObj->fd, buffer + headLen, leftLen); - //tTrace("%s TCP data is received, ip:%s port:%u len:%d", pTcp->label, pFdObj->ipstr, pFdObj->port, dataLen); + // tTrace("%s TCP data is received, ip:%s port:%u len:%d", pTcp->label, pFdObj->ipstr, pFdObj->port, dataLen); if (leftLen != retLen) { tError("%s read error, leftLen:%d retLen:%d", pTcp->label, leftLen, retLen); + tfree(buffer); taosCleanUpTcpFdObj(pFdObj); continue; } @@ -268,6 +285,7 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, short pFdObj->port = port; pFdObj->pTcp = pTcp; pFdObj->thandle = thandle; + pFdObj->signature = pFdObj; event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; event.data.ptr = pFdObj; diff --git a/src/rpc/src/ttcpserver.c b/src/rpc/src/ttcpserver.c index dbb8ec9fae..b6b0e07230 100644 --- a/src/rpc/src/ttcpserver.c +++ b/src/rpc/src/ttcpserver.c @@ -37,6 +37,7 @@ #endif typedef struct _fd_obj { + void *signature; int fd; // TCP socket FD void * thandle; // handle from upper layer, like TAOS char ipstr[TAOS_IPv4ADDR_LEN]; @@ -75,6 +76,7 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { SThreadObj *pThreadObj; if (pFdObj == NULL) return; + if (pFdObj->signature != pFdObj) return; pThreadObj = pFdObj->pThreadObj; if (pThreadObj == NULL) { @@ -274,6 +276,7 @@ void taosAcceptTcpConnection(void *arg) { pFdObj->ip = clientAddr.sin_addr.s_addr; pFdObj->port = htons(clientAddr.sin_port); pFdObj->pThreadObj = pThreadObj; + pFdObj->signature = pFdObj; event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; event.data.ptr = pFdObj; diff --git a/src/rpc/src/tudp.c b/src/rpc/src/tudp.c index b79f40019e..1940c89e85 100644 --- a/src/rpc/src/tudp.c +++ b/src/rpc/src/tudp.c @@ -27,7 +27,6 @@ #include "thash.h" #include "thaship.h" #include "tlog.h" -#include "tlog.h" #include "tsocket.h" #include "tsystem.h" #include "ttimer.h" @@ -50,9 +49,9 @@ typedef struct { char label[12]; // copy from udpConnSet; pthread_t thread; pthread_mutex_t mutex; - void * tmrCtrl; // copy from UdpConnSet; + void * tmrCtrl; // copy from UdpConnSet; void * hash; - void * shandle; // handle passed by upper layer during server initialization + void * shandle; // handle passed by upper layer during server initialization void * pSet; void *(*processData)(char *data, int dataLen, unsigned int ip, short port, void *shandle, void *thandle, void *chandle); @@ -154,8 +153,8 @@ void *taosReadTcpData(void *argv) { pInfo->msgLen = (int32_t)htonl((uint32_t)pInfo->msgLen); tinet_ntoa(ipstr, pMonitor->ip); - tTrace("%s receive packet via TCP:%s:%d, msgLen:%d, handle:0x%x, source:0x%08x dest:0x%08x tranId:%d", - pSet->label, ipstr, pInfo->port, pInfo->msgLen, pInfo->handle, pHead->sourceId, pHead->destId, pHead->tranId); + tTrace("%s receive packet via TCP:%s:%d, msgLen:%d, handle:0x%x, source:0x%08x dest:0x%08x tranId:%d", pSet->label, + ipstr, pInfo->port, pInfo->msgLen, pInfo->handle, pHead->sourceId, pHead->destId, pHead->tranId); fd = taosOpenTcpClientSocket(ipstr, (int16_t)pInfo->port, tsLocalIp); if (fd < 0) { @@ -174,12 +173,21 @@ void *taosReadTcpData(void *argv) { } else { tTrace("%s handle:0x%x is sent to server", pSet->label, pInfo->handle); char *buffer = malloc((size_t)pInfo->msgLen); - retLen = taosReadMsg(fd, buffer, pInfo->msgLen); + if (NULL == buffer) { + tError("%s failed to malloc(size:%d) for recv server data", pSet->label, pInfo->msgLen); + retLen = 0; + //taosCloseTcpSocket(fd); + //pMonitor->pSet = NULL; + //return NULL; + } else { + retLen = taosReadMsg(fd, buffer, pInfo->msgLen); + } + pMonitor->pSet = NULL; if (retLen != pInfo->msgLen) { tError("%s failed to read data from server, msgLen:%d retLen:%d", pSet->label, pInfo->msgLen, retLen); - free(buffer); + tfree(buffer); } else { (*pSet->fp)(buffer, pInfo->msgLen, pMonitor->ip, (int16_t)pInfo->port, pSet->shandle, NULL, pMonitor->pConn); } @@ -197,8 +205,8 @@ int taosReceivePacketViaTcp(uint32_t ip, STaosHeader *pHead, SUdpConn *pConn) { pthread_attr_t thattr; pthread_t thread; - tTrace("%s receive packet via TCP, handle:0x%x, source:0x%08x dest:0x%08x tranId:%d", - pSet->label, pInfo->handle, pHead->sourceId, pHead->destId, pHead->tranId); + tTrace("%s receive packet via TCP, handle:0x%x, source:0x%08x dest:0x%08x tranId:%d", pSet->label, pInfo->handle, + pHead->sourceId, pHead->destId, pHead->tranId); SMonitor *pMonitor = (SMonitor *)calloc(1, sizeof(SMonitor)); pMonitor->dataLen = sizeof(STaosHeader) + sizeof(SPacketInfo); @@ -254,8 +262,8 @@ void *taosRecvUdpData(void *param) { STaosHeader *pHead = (STaosHeader *)msg; msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); if (leftLen < minSize || msgLen > leftLen || msgLen < minSize) { - tError("%s msg is messed up, dataLen:%d processedLen:%d count:%d msgLen:%d", - pConn->label, dataLen, processedLen, count, msgLen); + tError("%s msg is messed up, dataLen:%d processedLen:%d count:%d msgLen:%d", pConn->label, dataLen, + processedLen, count, msgLen); break; } @@ -314,6 +322,12 @@ void *taosTransferDataViaTcp(void *argv) { tError("%s failed to read msg header, retLen:%d", pSet->label, retLen); } else { SMonitor *pMonitor = (SMonitor *)calloc(1, sizeof(SMonitor)); + if (NULL == pMonitor) { + tError("%s malloc failed by TransferViaTcp from client", pSet->label); + taosCloseSocket(connFd); + free(pTransfer); + return NULL; + } pMonitor->dataLen = sizeof(STaosHeader); memcpy(pMonitor->data, &head, (size_t)pMonitor->dataLen); ((STaosHeader *)pMonitor->data)->msgLen = (int32_t)htonl(sizeof(STaosHeader)); @@ -325,13 +339,20 @@ void *taosTransferDataViaTcp(void *argv) { msgLen = (int32_t)htonl((uint32_t)head.msgLen); char *buffer = malloc((size_t)msgLen); + if (NULL == buffer) { + tError("%s malloc failed for msg by TransferViaTcp", pSet->label); + taosCloseSocket(connFd); + free(pTransfer); + return NULL; + } + leftLen = msgLen - (int)sizeof(STaosHeader); retLen = taosReadMsg(connFd, buffer + sizeof(STaosHeader), leftLen); pMonitor->pSet = NULL; if (retLen != leftLen) { - tError("%s failed to read data from client, leftLen:%d retLen:%d, error:%s", - pSet->label, leftLen, retLen, strerror(errno)); + tError("%s failed to read data from client, leftLen:%d retLen:%d, error:%s", pSet->label, leftLen, retLen, + strerror(errno)); } else { tTrace("%s data is received from client via TCP from 0x%x:%d, msgLen:%d", pSet->label, pTransfer->ip, pTransfer->port, msgLen); @@ -353,6 +374,12 @@ void *taosTransferDataViaTcp(void *argv) { tError("%s invalid handle:%p, connection shall be closed", pSet->label, pHeader); } else { SMonitor *pMonitor = (SMonitor *)calloc(1, sizeof(SMonitor)); + if (NULL == pMonitor) { + tError("%s malloc failed by TransferViaTcp to client", pSet->label); + taosCloseSocket(connFd); + free(pTransfer); + return NULL; + } pMonitor->dataLen = sizeof(STaosHeader); memcpy(pMonitor->data, (void *)handle, (size_t)pMonitor->dataLen); STaosHeader *pThead = (STaosHeader *)pMonitor->data; @@ -370,8 +397,8 @@ void *taosTransferDataViaTcp(void *argv) { if (retLen != msgLen) { tError("%s failed to send data to client, msgLen:%d retLen:%d", pSet->label, msgLen, retLen); } else { - tTrace("%s data is sent to client successfully via TCP to 0x%x:%d, size:%d", - pSet->label, pTransfer->ip, pTransfer->port, msgLen); + tTrace("%s data is sent to client successfully via TCP to 0x%x:%d, size:%d", pSet->label, pTransfer->ip, + pTransfer->port, msgLen); } } } @@ -462,8 +489,7 @@ void *taosInitUdpConnection(char *ip, short port, char *label, int threads, void sprintf(udplabel, "%s.b", label); pSet->tmrCtrl = taosTmrInit(RPC_MAX_UDP_CONNS * threads, 5, 5000, udplabel); if (pSet->tmrCtrl == NULL) { - tError("%s failed to initialize tmrCtrl") - taosCleanUpUdpConnection(pSet); + tError("%s failed to initialize tmrCtrl") taosCleanUpUdpConnection(pSet); return NULL; } // } @@ -669,8 +695,8 @@ int taosSendPacketViaTcp(uint32_t ip, short port, char *data, int dataLen, void msgLen = sizeof(STaosHeader) + sizeof(SPacketInfo); pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); code = taosSendUdpData(ip, port, buffer, msgLen, chandle); - tTrace("%s data from server will be sent via TCP:%d, msgType:%d, length:%d, handle:0x%x", - pSet->label, pInfo->port, pHead->msgType, htonl((uint32_t)pInfo->msgLen), pInfo->handle); + tTrace("%s data from server will be sent via TCP:%d, msgType:%d, length:%d, handle:0x%x", pSet->label, pInfo->port, + pHead->msgType, htonl((uint32_t)pInfo->msgLen), pInfo->handle); if (code > 0) code = dataLen; } else { // send from client diff --git a/src/sdb/CMakeLists.txt b/src/sdb/CMakeLists.txt old mode 100755 new mode 100644 index 3dda245cd2..76b407fded --- a/src/sdb/CMakeLists.txt +++ b/src/sdb/CMakeLists.txt @@ -1,10 +1,15 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -IF (TD_LINUX) - AUX_SOURCE_DIRECTORY(src SRC) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/src/inc ${TD_OS_DIR}/inc inc) - ADD_LIBRARY(sdb ${SRC}) - TARGET_LINK_LIBRARIES(sdb trpc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) +INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) +INCLUDE_DIRECTORIES(inc) + +IF (TD_LINUX_64) + AUX_SOURCE_DIRECTORY(src SRC) + ADD_LIBRARY(sdb ${SRC}) + TARGET_LINK_LIBRARIES(sdb trpc) + IF (TD_CLUSTER) + TARGET_LINK_LIBRARIES(sdb sdb_cluster) + ENDIF () ENDIF () \ No newline at end of file diff --git a/src/sdb/src/sdbEngine.c b/src/sdb/src/sdbEngine.c index 47d04b1168..59e3d7e039 100644 --- a/src/sdb/src/sdbEngine.c +++ b/src/sdb/src/sdbEngine.c @@ -35,6 +35,8 @@ extern char version[]; const int16_t sdbFileVersion = 0; +int sdbExtConns = 0; +int sdbMaster = 1; void *(*sdbInitIndexFp[])(int maxRows, int dataSize) = {sdbOpenStrHash, sdbOpenIntHash, sdbOpenIntHash}; @@ -372,60 +374,65 @@ int64_t sdbInsertRow(void *handle, void *row, int rowSize) { pthread_mutex_lock(&pTable->mutex); - pTable->id++; - sdbVersion++; - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - // TODO: here need to change - *((uint32_t *)pObj) = ++pTable->autoIndex; - (*(pTable->appTool))(SDB_TYPE_ENCODE, pObj, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - } + if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_INSERT, rowHead->data, rowHead->rowSize) == 0) { + pTable->id++; + sdbVersion++; + if (pTable->keyType == SDB_KEYTYPE_AUTO) { + // TODO: here need to change + *((uint32_t *)pObj) = ++pTable->autoIndex; + (*(pTable->appTool))(SDB_TYPE_ENCODE, pObj, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); + } - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum while inserting, sdb: %s", pTable->name); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } + rowHead->delimiter = SDB_DELIMITER; + rowHead->id = pTable->id; + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("failed to get checksum while inserting, sdb: %s", pTable->name); + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } - // update in SDB layer - rowMeta.id = pTable->id; - rowMeta.offset = pTable->size; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = pObj; - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pObj, &rowMeta); + // update in SDB layer + rowMeta.id = pTable->id; + rowMeta.offset = pTable->size; + rowMeta.rowSize = rowHead->rowSize; + rowMeta.row = pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pObj, &rowMeta); + + /* Update the disk content */ + /* write(pTable->fd, &action, sizeof(action)); */ + /* pTable->size += sizeof(action); */ + twrite(pTable->fd, rowHead, real_size); + pTable->size += real_size; + sdbFinishCommit(pTable); - /* Update the disk content */ - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - twrite(pTable->fd, rowHead, real_size); - pTable->size += real_size; - sdbFinishCommit(pTable); + sdbAddIntoUpdateList(pTable, SDB_TYPE_INSERT, rowMeta.row); - sdbAddIntoUpdateList(pTable, SDB_TYPE_INSERT, rowMeta.row); + pTable->numOfRows++; + switch (pTable->keyType) { + case SDB_KEYTYPE_STRING: + sdbTrace( + "table:%s, a record is inserted:%s, sdbVersion:%ld id:%ld rowSize:%d numOfRows:%d fileSize:%ld", + pTable->name, (char *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); + break; + case SDB_KEYTYPE_UINT32: + case SDB_KEYTYPE_AUTO: + sdbTrace( + "table:%s, a record is inserted:%d, sdbVersion:%ld id:%ld rowSize:%d numOfRows:%d fileSize:%ld", + pTable->name, *(int32_t *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); + break; + default: + sdbTrace( + "table:%s, a record is inserted, sdbVersion:%ld id:%ld rowSize:%d numOfRows:%d fileSize:%ld", + pTable->name, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); + break; + } - pTable->numOfRows++; - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%ld id:%ld rowSize:%d numOfRows:%d fileSize:%ld", - pTable->name, (char *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - case SDB_KEYTYPE_UINT32: - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is inserted:%d, sdbVersion:%ld id:%ld rowSize:%d numOfRows:%d fileSize:%ld", - pTable->name, *(int32_t *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - default: - sdbTrace("table:%s, a record is inserted, sdbVersion:%ld id:%ld rowSize:%d numOfRows:%d fileSize:%ld", - pTable->name, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; + id = rowMeta.id; } - id = rowMeta.id; - tfree(rowHead); pthread_mutex_unlock(&pTable->mutex); @@ -482,48 +489,52 @@ int sdbDeleteRow(void *handle, void *row) { pthread_mutex_lock(&pTable->mutex); - pTable->id++; - sdbVersion++; - - rowHead->delimiter = SDB_DELIMITER; - rowHead->rowSize = rowSize; - rowHead->id = -(pTable->id); - memcpy(rowHead->data, row, rowSize); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { - sdbError("failed to get checksum while inserting, sdb: %s", pTable->name); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - twrite(pTable->fd, rowHead, total_size); - pTable->size += total_size; - sdbFinishCommit(pTable); - - pTable->numOfRows--; - // TODO: Change the update list here - sdbAddIntoUpdateList(pTable, SDB_TYPE_DELETE, pMetaRow); - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%ld id:%ld numOfRows:%d", - pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_UINT32: - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is deleted:%d, sdbVersion:%ld id:%ld numOfRows:%d", - pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - default: - sdbTrace("table:%s, a record is deleted, sdbVersion:%ld id:%ld numOfRows:%d", pTable->name, sdbVersion, - pTable->id, pTable->numOfRows); - break; - } + if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_DELETE, (char *)row, rowSize) == 0) { + pTable->id++; + sdbVersion++; + + rowHead->delimiter = SDB_DELIMITER; + rowHead->rowSize = rowSize; + rowHead->id = -(pTable->id); + memcpy(rowHead->data, row, rowSize); + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { + sdbError("failed to get checksum while inserting, sdb: %s", pTable->name); + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + /* write(pTable->fd, &action, sizeof(action)); */ + /* pTable->size += sizeof(action); */ + twrite(pTable->fd, rowHead, total_size); + pTable->size += total_size; + sdbFinishCommit(pTable); + + pTable->numOfRows--; + // TODO: Change the update list here + sdbAddIntoUpdateList(pTable, SDB_TYPE_DELETE, pMetaRow); + switch (pTable->keyType) { + case SDB_KEYTYPE_STRING: + sdbTrace( + "table:%s, a record is deleted:%s, sdbVersion:%ld id:%ld numOfRows:%d", + pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); + break; + case SDB_KEYTYPE_UINT32: + case SDB_KEYTYPE_AUTO: + sdbTrace( + "table:%s, a record is deleted:%d, sdbVersion:%ld id:%ld numOfRows:%d", + pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); + break; + default: + sdbTrace("table:%s, a record is deleted, sdbVersion:%ld id:%ld numOfRows:%d", + pTable->name, sdbVersion, pTable->id, pTable->numOfRows); + break; + } - // Delete from current layer - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, row); + // Delete from current layer + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, row); - code = 0; + code = 0; + } pthread_mutex_unlock(&pTable->mutex); @@ -578,47 +589,51 @@ int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated) { pthread_mutex_lock(&pTable->mutex); - pTable->id++; - sdbVersion++; + if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_UPDATE, rowHead->data, rowHead->rowSize) == 0) { + pTable->id++; + sdbVersion++; - // write to the new position - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum, sdb: %s id: %d", pTable->name, rowHead->id); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - twrite(pTable->fd, rowHead, real_size); + // write to the new position + rowHead->delimiter = SDB_DELIMITER; + rowHead->id = pTable->id; + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("failed to get checksum, sdb: %s id: %d", pTable->name, rowHead->id); + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + /* write(pTable->fd, &action, sizeof(action)); */ + /* pTable->size += sizeof(action); */ + twrite(pTable->fd, rowHead, real_size); - pMeta->id = pTable->id; - pMeta->offset = pTable->size; - pMeta->rowSize = rowHead->rowSize; - pTable->size += real_size; + pMeta->id = pTable->id; + pMeta->offset = pTable->size; + pMeta->rowSize = rowHead->rowSize; + pTable->size += real_size; - sdbFinishCommit(pTable); + sdbFinishCommit(pTable); - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is updated:%s, sdbVersion:%ld id:%ld numOfRows:%d", - pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_UINT32: - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is updated:%d, sdbVersion:%ld id:%ld numOfRows:%d", - pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - default: - sdbTrace("table:%s, a record is updated, sdbVersion:%ld id:%ld numOfRows:%d", pTable->name, sdbVersion, - pTable->id, pTable->numOfRows); - break; - } + switch (pTable->keyType) { + case SDB_KEYTYPE_STRING: + sdbTrace( + "table:%s, a record is updated:%s, sdbVersion:%ld id:%ld numOfRows:%d", + pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); + break; + case SDB_KEYTYPE_UINT32: + case SDB_KEYTYPE_AUTO: + sdbTrace( + "table:%s, a record is updated:%d, sdbVersion:%ld id:%ld numOfRows:%d", + pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); + break; + default: + sdbTrace("table:%s, a record is updated, sdbVersion:%ld id:%ld numOfRows:%d", pTable->name, sdbVersion, + pTable->id, pTable->numOfRows); + break; + } - sdbAddIntoUpdateList(pTable, SDB_TYPE_UPDATE, pMetaRow); - code = 0; + sdbAddIntoUpdateList(pTable, SDB_TYPE_UPDATE, pMetaRow); + code = 0; + } pthread_mutex_unlock(&pTable->mutex); @@ -652,43 +667,47 @@ int sdbBatchUpdateRow(void *handle, void *row, int rowSize) { } pthread_mutex_lock(&pTable->mutex); - - (*(pTable->appTool))(SDB_TYPE_BEFORE_BATCH_UPDATE, pMetaRow, NULL, 0, NULL); - - void *next_row = pMetaRow; - while (next_row != NULL) { - pTable->id++; - sdbVersion++; - - void *last_row = next_row; - next_row = (*(pTable->appTool))(SDB_TYPE_BATCH_UPDATE, last_row, (char *)row, rowSize, 0); - memset(rowHead, 0, sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM)); - - // update in current layer - pMeta->id = pTable->id; - pMeta->offset = pTable->size; - - // write to disk - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pMeta->id; - (*(pTable->appTool))(SDB_TYPE_ENCODE, last_row, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - taosCalcChecksumAppend(0, (uint8_t *)rowHead, sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - pMeta->rowSize = rowHead->rowSize; - lseek(pTable->fd, pTable->size, SEEK_SET); - twrite(pTable->fd, rowHead, sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - pTable->size += (sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - - sdbAddIntoUpdateList(pTable, SDB_TYPE_UPDATE, last_row); - - if (next_row != NULL) { - pMeta = sdbGetRowMeta(handle, next_row); + if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_BATCH_UPDATE, row, rowSize) == 0) { + /* // write action */ + /* write(pTable->fd, &action, sizeof(action)); */ + /* pTable->size += sizeof(action); */ + + (*(pTable->appTool))(SDB_TYPE_BEFORE_BATCH_UPDATE, pMetaRow, NULL, 0, NULL); + + void *next_row = pMetaRow; + while (next_row != NULL) { + pTable->id++; + sdbVersion++; + + void *last_row = next_row; + next_row = (*(pTable->appTool))(SDB_TYPE_BATCH_UPDATE, last_row, (char *)row, rowSize, 0); + memset(rowHead, 0, sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM)); + + // update in current layer + pMeta->id = pTable->id; + pMeta->offset = pTable->size; + + // write to disk + rowHead->delimiter = SDB_DELIMITER; + rowHead->id = pMeta->id; + (*(pTable->appTool))(SDB_TYPE_ENCODE, last_row, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); + taosCalcChecksumAppend(0, (uint8_t *)rowHead, sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); + pMeta->rowSize = rowHead->rowSize; + lseek(pTable->fd, pTable->size, SEEK_SET); + twrite(pTable->fd, rowHead, sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); + pTable->size += (sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); + + sdbAddIntoUpdateList(pTable, SDB_TYPE_UPDATE, last_row); + + if (next_row != NULL) { + pMeta = sdbGetRowMeta(handle, next_row); + } } - } - sdbFinishCommit(pTable); - - (*(pTable->appTool))(SDB_TYPE_AFTER_BATCH_UPDATE, pMetaRow, NULL, 0, NULL); + sdbFinishCommit(pTable); + (*(pTable->appTool))(SDB_TYPE_AFTER_BATCH_UPDATE, pMetaRow, NULL, 0, NULL); + } pthread_mutex_unlock(&pTable->mutex); tfree(rowHead); diff --git a/src/sdb/src/sdbstr.c b/src/sdb/src/sdbstr.c index e873507d23..90bf2a3d43 100644 --- a/src/sdb/src/sdbstr.c +++ b/src/sdb/src/sdbstr.c @@ -13,6 +13,31 @@ * along with this program. If not, see . */ +#include "sdbint.h" + char* sdbStatusStr[] = {"offline", "unsynced", "syncing", "serving", "null"}; char* sdbRoleStr[] = {"unauthed", "undecided", "master", "slave", "null"}; + +#ifndef CLUSTER + +/* + * Lite Version sync request is always successful + */ +int sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int dataLen) { + return 0; +} + +/* + * Lite Version does not need to initialize peers + */ +int sdbInitPeers(char *directory) { + return 0; +} + +/* + * Lite Version does not need to cleanup peers + */ +void sdbCleanUpPeers(){} + +#endif \ No newline at end of file diff --git a/src/system/CMakeLists.txt b/src/system/CMakeLists.txt old mode 100755 new mode 100644 index 3a6e052650..5c4ab62d24 --- a/src/system/CMakeLists.txt +++ b/src/system/CMakeLists.txt @@ -1,32 +1,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -IF (TD_LINUX) - - AUX_SOURCE_DIRECTORY(./src SRC) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/src/inc) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/src/client/inc) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/src/modules/http/inc) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/src/modules/monitor/inc) - INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc inc) - - ADD_EXECUTABLE(taosd ${SRC}) - TARGET_LINK_LIBRARIES(taosd taos_static trpc tutil sdb monitor pthread http) - - SET(PREPARE_ENV_CMD "prepare_env_cmd") - SET(PREPARE_ENV_TARGET "prepare_env_target") - ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} - POST_BUILD - COMMAND echo "make test directory" - DEPENDS taosd - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/ - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/ - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/ - COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMENT "prepare taosd environment") - ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) +ADD_SUBDIRECTORY(detail) +IF (TD_LITE) + ADD_SUBDIRECTORY(lite) ENDIF () \ No newline at end of file diff --git a/src/system/detail/CMakeLists.txt b/src/system/detail/CMakeLists.txt new file mode 100644 index 0000000000..9233d6c71b --- /dev/null +++ b/src/system/detail/CMakeLists.txt @@ -0,0 +1,42 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +IF (TD_LINUX_64) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/modules/http/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/modules/monitor/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/inc) + INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/util/cluster/inc) + INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) + INCLUDE_DIRECTORIES(inc) + AUX_SOURCE_DIRECTORY(./src SRC) + LIST(REMOVE_ITEM SRC ./src/vnodeFileUtil.c) + LIST(REMOVE_ITEM SRC ./src/taosGrant.c) + + ADD_EXECUTABLE(taosd ${SRC}) + + TARGET_LINK_LIBRARIES(taosd taos_static trpc tutil sdb monitor pthread http) + + IF (TD_LITE) + TARGET_LINK_LIBRARIES(taosd taosd_lite) + ELSE () + TARGET_LINK_LIBRARIES(taosd taosd_cluster) + ENDIF () + + SET(PREPARE_ENV_CMD "prepare_env_cmd") + SET(PREPARE_ENV_TARGET "prepare_env_target") + ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} + POST_BUILD + COMMAND echo "make test directory" + DEPENDS taosd + COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/ + COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/ + COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/ + COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMENT "prepare taosd environment") + ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) + +ENDIF () diff --git a/src/system/inc/dnodeSystem.h b/src/system/detail/inc/dnodeSystem.h similarity index 65% rename from src/system/inc/dnodeSystem.h rename to src/system/detail/inc/dnodeSystem.h index 22a90e76ed..632b723ac0 100644 --- a/src/system/inc/dnodeSystem.h +++ b/src/system/detail/inc/dnodeSystem.h @@ -22,7 +22,12 @@ extern "C" { #include -enum _module { TSDB_MOD_HTTP, TSDB_MOD_MONITOR, TSDB_MOD_MAX }; +#define tsetModuleStatus(mod) \ + { tsModuleStatus |= (1 << mod); } +#define tclearModuleStatus(mod) \ + { tsModuleStatus &= ~(1 << mod); } + +enum _module { TSDB_MOD_MGMT, TSDB_MOD_HTTP, TSDB_MOD_MONITOR, TSDB_MOD_MAX }; typedef struct { char *name; @@ -41,11 +46,25 @@ extern pthread_mutex_t dmutex; void dnodeCleanUpSystem(); int dnodeInitSystem(); +int dnodeInitSystemSpec(); +void dnodeStartModuleSpec(); +void dnodeParseParameterK(); +void dnodeProcessModuleStatus(uint32_t status); +void dnodeResetSystem(); +void dnodeCheckDbRunning(const char* dir); -int vnodeInitSystem(); +void vnodeCleanUpSystem(); +int vnodeInitSystem(); +void vnodeInitMgmtIp(); +void vnodeInitQHandle(); int mgmtInitSystem(); void mgmtCleanUpSystem(); +int mgmtStartSystem(); +void mgmtStopSystem(); + +int taosCreateTierDirectory(); +void taosCleanupTier(); #ifdef __cplusplus } diff --git a/src/system/inc/mgmt.h b/src/system/detail/inc/mgmt.h similarity index 80% rename from src/system/inc/mgmt.h rename to src/system/detail/inc/mgmt.h index 995ba2ecb5..1d1f325ec2 100644 --- a/src/system/inc/mgmt.h +++ b/src/system/detail/inc/mgmt.h @@ -24,6 +24,7 @@ extern "C" { #include #include #include +#include #include "sdb.h" #include "tglobalcfg.h" @@ -77,7 +78,8 @@ typedef struct { uint16_t numOfCores; // from dnode status msg uint8_t alternativeRole; // from dnode status msg, 0-any, 1-mgmt, 2-dnode uint8_t reserveStatus; - float memoryAvailable; // from dnode status msg + uint16_t numOfTotalVnodes; // from dnode status msg, config information + uint16_t unused; float diskAvailable; // from dnode status msg int32_t bandwidthMb; // config by user int16_t cpuAvgUsage; // calc from sys.cpu @@ -94,6 +96,7 @@ typedef struct { SVnodeLoad vload[TSDB_MAX_VNODES]; char reserved[16]; char updateEnd[1]; + void * thandle; } SDnodeObj; typedef struct { @@ -270,10 +273,11 @@ extern SDnodeObj dnodeObj; // dnodeInt API int mgmtInitDnodeInt(); void mgmtCleanUpDnodeInt(); -int mgmtSendCreateMsgToVnode(STabObj *pMeter, int vnode); -int mgmtSendRemoveMeterMsgToVnode(STabObj *pMeter, int vnode); -int mgmtSendVPeersMsg(SVgObj *pVgroup, SDbObj *pDb); -int mgmtSendFreeVnodeMsg(int vnode); +int mgmtSendCreateMsgToVgroup(STabObj *pMeter, SVgObj *pVgroup); +int mgmtSendRemoveMeterMsgToDnode(STabObj *pMeter, SVgObj *pVgroup); +int mgmtSendVPeersMsg(SVgObj *pVgroup); +int mgmtSendFreeVnodeMsg(SVgObj *pVgroup); +int mgmtSendOneFreeVnodeMsg(SVnodeGid *pVnodeGid); // shell API int mgmtInitShell(); @@ -281,13 +285,22 @@ void mgmtCleanUpShell(); int mgmtRetriveUserAuthInfo(char *user, char *spi, char *encrypt, uint8_t *secret, uint8_t *ckey); // acct API +int mgmtInitAccts(); +SAcctObj *mgmtGetAcct(char *name); +int mgmtCreateAcct(char *name, char *pass, SAcctCfg *pCfg); +int mgmtUpdateAcct(SAcctObj *pAcct); +int mgmtDropAcct(char *name); int mgmtAddDbIntoAcct(SAcctObj *pAcct, SDbObj *pDb); int mgmtRemoveDbFromAcct(SAcctObj *pAcct, SDbObj *pDb); int mgmtAddUserIntoAcct(SAcctObj *pAcct, SUserObj *pUser); int mgmtRemoveUserFromAcct(SAcctObj *pAcct, SUserObj *pUser); int mgmtAddConnIntoAcct(SConnObj *pConn); int mgmtRemoveConnFromAcct(SConnObj *pConn); -void mgmtCheckAcct(); +int mgmtGetAcctMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn); +int mgmtRetrieveAccts(SShowObj *pShow, char *data, int rows, SConnObj *pConn); +void mgmtCheckAcct(); +void mgmtCleanUpAccts(); +int mgmtAlterAcct(char *name, char *pass, SAcctCfg *pCfg); int64_t mgmtGetAcctStatistic(SAcctObj *pAcct); // user API @@ -333,18 +346,17 @@ void mgmtCleanUpDbs(); int mgmtInitVgroups(); SVgObj *mgmtGetVgroup(int vgId); SVgObj *mgmtCreateVgroup(SDbObj *pDb); -int mgmtDropVgroup(SDbObj *pDb, SVgObj *pVgroup); +int mgmtDropVgroup(SDbObj *pDb, SVgObj *pVgroup); void mgmtSetVgroupIdPool(); -int mgmtGetVgroupMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn); -int mgmtRetrieveVgroups(SShowObj *pShow, char *data, int rows, SConnObj *pConn); -void mgmtCleanUpVgroups(); -SAcctObj *mgmtGetVgroupAcct(int vgId); +int mgmtGetVgroupMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn); +int mgmtRetrieveVgroups(SShowObj *pShow, char *data, int rows, SConnObj *pConn); +void mgmtCleanUpVgroups(); // meter API int mgmtInitMeters(); STabObj *mgmtGetMeter(char *meterId); STabObj *mgmtGetMeterInfo(char *src, char *tags[]); -int mgmtRetrieveMetricMeta(void *thandle, char **pStart, STabObj *pMetric, SMetricMetaMsg *pInfo); +int mgmtRetrieveMetricMeta(void *thandle, char **pStart, SMetricMetaMsg *pInfo); int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate); int mgmtDropMeter(SDbObj *pDb, char *meterId, int ignore); int mgmtAlterMeter(SDbObj *pDb, SAlterTableMsg *pAlter); @@ -357,12 +369,45 @@ bool mgmtMeterCreateFromMetric(STabObj *pMeterObj); bool mgmtIsMetric(STabObj *pMeterObj); bool mgmtIsNormalMeter(STabObj *pMeterObj); -int mgmtGetDnodeMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn); -int mgmtRetrieveDnodes(SShowObj *pShow, char *data, int rows, SConnObj *pConn); -void mgmtSetDnodeVgid(int vnode, int vgId); -void mgmtUnSetDnodeVgid(int vnode); +// grant API +void grantActiveSystem(const char* cfgFile); +void grantSendMsgToMgmt(); +void grantReset(); +void grantUpdate(void *pGrant); +bool grantCheckExpired(); +void grantRestoreTimeSeries(uint32_t timeseries); +void grantAddTimeSeries(uint32_t timeseries); +int grantCheckTimeSeries(uint32_t timeseries); +void grantResetCurStorage(uint64_t totalStorage); +int grantCheckStorage(); +int grantCheckDatabases(); +int grantCheckUsers(); +int grantCheckAccts(); +int grantCheckDnodes(); +int grantCheckConns(); +int grantCheckStreams(); +int grantCheckCpuCores(); +int grantCheckQueryTime(); + +// dnode API +int mgmtInitDnodes(); +SDnodeObj *mgmtGetDnode(uint32_t ip); +int mgmtCreateDnode(uint32_t ip); +int mgmtDropDnode(SDnodeObj *pDnode); +int mgmtDropDnodeByIp(uint32_t ip); +int mgmtUpdateDnode(SDnodeObj *pDnode); +int mgmtGetNextVnode(SVnodeGid *pVnodeGid); +void mgmtSetDnodeVgid(SVnodeGid vnodeGid[], int numOfVnodes, int vgId); +void mgmtUnSetDnodeVgid(SVnodeGid vnodeGid[], int numOfVnodes); +int mgmtGetDnodeMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn); +int mgmtRetrieveDnodes(SShowObj *pShow, char *data, int rows, SConnObj *pConn); +void mgmtCleanUpDnodes(); +int mgmtSendCfgDnodeMsg(char *cont); void mgmtSetDnodeMaxVnodes(SDnodeObj *pDnode); +int mgmtGetMnodeMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn); +int mgmtRetrieveMnodes(SShowObj *pShow, char *data, int rows, SConnObj *pConn); + int mgmtGetModuleMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn); int mgmtRetrieveModules(SShowObj *pShow, char *data, int rows, SConnObj *pConn); @@ -378,15 +423,29 @@ int mgmtRetrieveScores(SShowObj *pShow, char *data, int rows, SConnObj *pConn); int grantGetGrantsMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn); int grantRetrieveGrants(SShowObj *pShow, char *data, int rows, SConnObj *pConn); +// dnode balance api +int mgmtInitBalance(); +void mgmtCleanupBalance(); +int mgmtAllocVnodes(SVgObj *pVgroup); +void mgmtSetDnodeShellRemoving(SDnodeObj *pDnode); +void mgmtSetDnodeUnRemove(SDnodeObj *pDnode); +void mgmtStartBalanceTimer(int mseconds); +void mgmtSetDnodeOfflineOnSdbChanged(); +void mgmtUpdateVgroupState(SVgObj *pVgroup, int lbState, int srcIp); +bool mgmtAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode); + void mgmtSetModuleInDnode(SDnodeObj *pDnode, int moduleType); int mgmtUnSetModuleInDnode(SDnodeObj *pDnode, int moduleType); extern int (*mgmtGetMetaFp[])(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn); extern int (*mgmtRetrieveFp[])(SShowObj *pShow, char *data, int rows, SConnObj *pConn); +extern int tsDnodeUpdateSize; extern int tsVgUpdateSize; extern int tsDbUpdateSize; extern int tsUserUpdateSize; +extern int tsAcctUpdateSize; +extern int tsMnodeUpdateSize; #ifdef __cplusplus } diff --git a/src/system/detail/inc/mgmtBalance.h b/src/system/detail/inc/mgmtBalance.h new file mode 100644 index 0000000000..4157458a7f --- /dev/null +++ b/src/system/detail/inc/mgmtBalance.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_MGMTBALANCE_H +#define TDENGINE_MGMTBALANCE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#include "dnodeSystem.h" +#include "mgmt.h" +#include "tglobalcfg.h" +#include "tstatus.h" +#include "ttime.h" + +enum { + LB_DNODE_STATE_BALANCED, + LB_DNODE_STATE_BALANCING, + LB_DNODE_STATE_OFFLINE_REMOVING, + LB_DNODE_STATE_SHELL_REMOVING +}; + +enum { LB_VGROUP_STATE_READY, LB_VGROUP_STATE_UPDATE }; + +void mgmtCreateDnodeOrderList(); + +void mgmtReleaseDnodeOrderList(); + +void mgmtMakeDnodeOrderList(); + +void mgmtCalcSystemScore(); + +float mgmtTryCalcDnodeScore(SDnodeObj *pDnode, int extraVnode); + +bool mgmtCheckDnodeInOfflineState(SDnodeObj *pDnode); + +bool mgmtCheckDnodeInRemoveState(SDnodeObj *pDnode); + +bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int moduleType); + +void mgmtMonitorDnodeModule(); + +void mgmtSetModuleInDnode(SDnodeObj *pDnode, int moduleType); + +int mgmtUnSetModuleInDnode(SDnodeObj *pDnode, int moduleType); + +void mgmtMonitorVgroups(); + +void mgmtMonitorDnodes(); + +void mgmtCalcNumOfFreeVnodes(SDnodeObj *pDnode); + +extern void * dnodeSdb; +extern void * vgSdb; +extern void * balanceTimer; +extern int mgmtOrderedDnodesSize; +extern int mgmtOrderedDnodesMallocSize; +extern SDnodeObj **mgmtOrderedDnodes; +extern uint32_t mgmtAccessSquence; +extern SMgmtIpList mgmtIpList; + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_MGMTBALANCE_H diff --git a/src/system/inc/mgmtProfile.h b/src/system/detail/inc/mgmtProfile.h similarity index 100% rename from src/system/inc/mgmtProfile.h rename to src/system/detail/inc/mgmtProfile.h diff --git a/src/system/inc/mgmtSystem.h b/src/system/detail/inc/mgmtSystem.h similarity index 70% rename from src/system/inc/mgmtSystem.h rename to src/system/detail/inc/mgmtSystem.h index 818c2a6ce5..1262d7e834 100644 --- a/src/system/inc/mgmtSystem.h +++ b/src/system/detail/inc/mgmtSystem.h @@ -22,11 +22,29 @@ extern "C" { #include -void mgmtCleanUpSystem(); +int mgmtInitRedirect(); + +void mgmtCleanUpRedirect(); + +void*mgmtRedirectAllMsgs(char *msg, void *ahandle, void *thandle); + +void mgmtSdbWorkAsMasterCallback(); + +void mgmtSetDnodeOfflineOnSdbChanged(); void mgmtPrintSystemInfo(); -int32_t mgmtInitSystem(); +int mgmtInitSystem(); + +int mgmtStartCheckMgmtRunning(); + +void mgmtDoStatistic(void *handle, void *tmrId); + +void mgmtStartMgmtTimer(); + +void mgmtStopSystem(); + +void mgmtCleanUpSystem(); #ifdef __cplusplus } diff --git a/src/system/detail/inc/mgmtUtil.h b/src/system/detail/inc/mgmtUtil.h new file mode 100644 index 0000000000..aecb229dba --- /dev/null +++ b/src/system/detail/inc/mgmtUtil.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */#include +#include +#include + +#include "tast.h" + +#ifndef TBASE_MGMTUTIL_H +#define TBASE_MGMTUTIL_H + +typedef struct SSyntaxTreeFilterSupporter { + SSchema* pTagSchema; + int32_t numOfTags; + int32_t optr; +} SSyntaxTreeFilterSupporter; + +char* mgmtMeterGetTag(STabObj* pMeter, int32_t col, SSchema* pTagColSchema); +int32_t mgmtFindTagCol(STabObj * pMetric, const char * tagName); + +int32_t mgmtGetTagsLength(STabObj* pMetric, int32_t col); + +int32_t mgmtRetrieveMetersFromMetric(SMetricMetaMsg* pInfo, int32_t tableIndex, tQueryResultset* pRes); +int32_t mgmtDoJoin(SMetricMetaMsg* pMetricMetaMsg, tQueryResultset* pRes); +void mgmtReorganizeMetersInMetricMeta(SMetricMetaMsg* pInfo, int32_t index, tQueryResultset* pRes); + +bool tSkipListNodeFilterCallback(struct tSkipListNode *pNode, void *param); + +#endif //TBASE_MGMTUTIL_H diff --git a/src/system/inc/vnode.h b/src/system/detail/inc/vnode.h similarity index 86% rename from src/system/inc/vnode.h rename to src/system/detail/inc/vnode.h index 13461278dc..4d2ebdfa35 100644 --- a/src/system/inc/vnode.h +++ b/src/system/detail/inc/vnode.h @@ -40,6 +40,7 @@ extern "C" { #include "tutil.h" #include "vnodeCache.h" #include "vnodeFile.h" +#include "vnodePeer.h" #include "vnodeShell.h" #define TSDB_FILE_HEADER_LEN 512 @@ -87,12 +88,14 @@ typedef struct { char data[]; } SData; +#pragma pack(push, 8) typedef struct { - int vnode; - SVnodeCfg cfg; + SVnodeStatisticInfo vnodeStatistic; + int vnode; + SVnodeCfg cfg; // SDiskDesc tierDisk[TSDB_MAX_TIER]; SVPeerDesc vpeers[TSDB_VNODES_SUPPORT]; - SVnodeStatisticInfo vnodeStatistic; + SVnodePeer * peerInfo[TSDB_VNODES_SUPPORT]; char selfIndex; char status; char accessState; // Vnode access state, Readable/Writable @@ -159,6 +162,7 @@ typedef struct { TAOS * dbConn; SMeterObjHeader *meterIndex; } SVnodeObj; +#pragma pack(pop) typedef struct SColumn { short colId; @@ -208,13 +212,9 @@ typedef struct { } SVMsgHeader; /* - * the value of QInfo.signature is used to denote that a query is executing, it - * isn't safe - * to release QInfo yet. - * The release operations will be blocked in a busy-waiting until the query - * operation reach a safepoint. - * Then it will reset the signature in a atomic operation, followed by release - * operation. + * The value of QInfo.signature is used to denote that a query is executing, it isn't safe to release QInfo yet. + * The release operations will be blocked in a busy-waiting until the query operation reach a safepoint. + * Then it will reset the signature in a atomic operation, followed by release operation. * Only the QInfo.signature == QInfo, this structure can be released safely. */ #define TSDB_QINFO_QUERY_FLAG 0x1 @@ -233,28 +233,36 @@ typedef struct { struct tSQLBinaryExpr; -typedef struct SColumnFilter { - SColumnFilterMsg data; - int16_t colIdx; - int16_t colIdxInBuf; +typedef struct SColumnInfoEx { + SColumnInfo data; + int16_t colIdx; + int16_t colIdxInBuf; /* * 0: denotes if its is required in the first round of scan of data block * 1: denotes if its is required in the secondary scan */ int16_t req[2]; -} SColumnFilter; +} SColumnInfoEx; -typedef bool (*__filter_func_t)(SColumnFilter *pFilter, char *val1, char *val2); +struct SColumnFilterElem; -typedef struct SColumnFilterInfo { - SColumnFilter pFilter; - int16_t elemSize; // element size in pData - __filter_func_t fp; // filter function - char * pData; // raw data, as the input for filter function -} SColumnFilterInfo; +typedef bool (*__filter_func_t)(struct SColumnFilterElem *pFilter, char *val1, char *val2); -typedef struct { +typedef struct SColumnFilterElem { + int16_t bytes; // column length + __filter_func_t fp; + SColumnFilterInfo filterInfo; +} SColumnFilterElem; + +typedef struct SSingleColumnFilterInfo { + SColumnInfoEx info; + int32_t numOfFilters; + SColumnFilterElem *pFilters; + char * pData; +} SSingleColumnFilterInfo; + +typedef struct SQuery { short numOfCols; SOrderVal order; char keyIsMet; // if key is met, it will be set @@ -278,26 +286,23 @@ typedef struct { TSKEY ekey; int64_t nAggTimeInterval; char intervalTimeUnit; // interval data type, used for daytime revise - char precision; + int8_t precision; int16_t numOfOutputCols; int16_t interpoType; int16_t checkBufferInLoop; // check if the buffer is full during scan each block - SLimitVal limit; int32_t rowSize; int32_t dataRowSize; // row size of each loaded data from disk, the value is - // used for prepare buffer - SSqlGroupbyExpr * pGroupbyExpr; - SSqlFunctionExpr *pSelectExpr; - - SColumnFilter *colList; - int32_t numOfFilterCols; - SColumnFilterInfo *pFilterInfo; - - int64_t *defaultVal; + // used for prepare buffer + SSqlGroupbyExpr * pGroupbyExpr; + SSqlFunctionExpr * pSelectExpr; + SColumnInfoEx * colList; + int32_t numOfFilterCols; + SSingleColumnFilterInfo *pFilterInfo; + int64_t * defaultVal; + TSKEY lastKey; - TSKEY lastKey; // buffer info int64_t pointsRead; // the number of points returned int64_t pointsToRead; // maximum number of points to read @@ -327,9 +332,10 @@ extern uint32_t tsRebootTime; extern void ** rpcQhandle; extern void * dmQhandle; extern void * queryQhandle; +extern int tsVnodePeers; extern int tsMaxVnode; +extern int tsMaxQueues; extern int tsOpenVnodes; -extern int tsMaxVnode; extern SVnodeObj *vnodeList; extern void * vnodeTmrCtrl; @@ -346,13 +352,13 @@ void *vnodeQueryOnMultiMeters(SMeterObj **pMeterObj, SSqlGroupbyExpr *pGroupbyEx SSqlGroupbyExpr *vnodeCreateGroupbyExpr(SQueryMeterMsg *pQuery, int32_t *code); SSqlFunctionExpr *vnodeCreateSqlFunctionExpr(SQueryMeterMsg *pQuery, int32_t *code); -bool vnodeValidateExprColumnInfo(SQueryMeterMsg* pQueryMsg, SSqlFuncExprMsg* pExprMsg); +bool vnodeValidateExprColumnInfo(SQueryMeterMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg); bool vnodeIsValidVnodeCfg(SVnodeCfg *pCfg); int32_t vnodeGetResultSize(void *handle, int32_t *numOfRows); -int32_t vnodeCopyQueryResultToMsg(void *handle, char *data, int32_t numOfRows); +int32_t vnodeCopyQueryResultToMsg(void *handle, char *data, int32_t numOfRows, int32_t *size); int64_t vnodeGetOffsetVal(void *thandle); @@ -360,7 +366,7 @@ bool vnodeHasRemainResults(void *handle); int vnodeRetrieveQueryResult(void *handle, int *pNum, char *argv[]); -int vnodeSaveQueryResult(void *handle, char *data); +int vnodeSaveQueryResult(void *handle, char *data, int32_t* size); int vnodeRetrieveQueryInfo(void *handle, int *numOfRows, int *rowSize, int16_t *timePrec); @@ -420,6 +426,10 @@ void vnodeCommitOver(SVnodeObj *pVnode); TSKEY vnodeGetFirstKey(int vnode); +int vnodeSyncRetrieveCache(int vnode, int fd); + +int vnodeSyncRestoreCache(int vnode, int fd); + pthread_t vnodeCreateCommitThread(SVnodeObj *pVnode); void vnodeCancelCommit(SVnodeObj *pVnode); @@ -445,6 +455,10 @@ void *vnodeCommitToFile(void *param); void *vnodeCommitMultiToFile(SVnodeObj *pVnode, int ssid, int esid); +int vnodeSyncRetrieveFile(int vnode, int fd, uint32_t fileId, uint64_t *fmagic); + +int vnodeSyncRestoreFile(int vnode, int sfd); + int vnodeWriteBlockToFile(SMeterObj *pObj, SCompBlock *pBlock, SData *data[], SData *cdata[], int pointsRead); int vnodeSearchPointInFile(SMeterObj *pObj, SQuery *pQuery); @@ -458,6 +472,8 @@ void vnodeCloseCommitFiles(SVnodeObj *pVnode); int vnodeReadLastBlockToMem(SMeterObj *pObj, SCompBlock *pBlock, SData *sdata[]); // vnode API +void vnodeUpdateStreamRole(SVnodeObj *pVnode); + int vnodeInitPeer(int numOfThreads); void vnodeCleanUpPeer(); @@ -470,8 +486,14 @@ void *vnodeGetMeterPeerConnection(SMeterObj *pObj, int index); int vnodeForwardToPeer(SMeterObj *pObj, char *msg, int msgLen, char action, int sversion); +void vnodeCloseAllSyncFds(int vnode); + void vnodeConfigVPeers(int vnode, int numOfPeers, SVPeerDesc peerDesc[]); +void vnodeStartSyncProcess(SVnodeObj *pVnode); + +void vnodeCancelSync(int vnode); + void vnodeListPeerStatus(char *buffer); void vnodeCheckOwnStatus(SVnodeObj *pVnode); @@ -506,6 +528,25 @@ int vnodeOpenShellVnode(int vnode); void vnodeCloseShellVnode(int vnode); +// memter mgmt +int vnodeInitMeterMgmt(); + +void vnodeCleanUpMeterMgmt(); + +int vnodeOpenMeterMgmtVnode(int vnode); + +int vnodeOpenMeterMgmtStoreVnode(int vnode); + +void vnodeCloseMeterMgmtVnode(int vnode); + +int vnodeCreateMeterMgmt(SMeterObj *pObj, SConnSec *pSec); + +void vnodeRemoveMeterMgmt(SMeterObj *pObj); + +SConnSec *vnodeGetMeterSec(int vnode, int sid); + +int vnodeCreateMeterObjFile(int vnode); + // mgmt int vnodeInitMgmt(); @@ -521,12 +562,14 @@ int vnodeRestoreMissedRemoveMsg(int vnode, int fd); int vnodeProcessBufferedCreateMsgs(int vnode); -int vnodeSendVpeerCfgMsg(int vnode); +void vnodeSendVpeerCfgMsg(int vnode); int vnodeSendMeterCfgMsg(int vnode, int sid); int vnodeMgmtConns(); +void vnodeRemoveFile(int vnode, int fileId); + // commit int vnodeInitCommit(int vnode); diff --git a/src/system/inc/vnodeCache.h b/src/system/detail/inc/vnodeCache.h similarity index 94% rename from src/system/inc/vnodeCache.h rename to src/system/detail/inc/vnodeCache.h index feb39e5703..8fb6d15647 100644 --- a/src/system/inc/vnodeCache.h +++ b/src/system/detail/inc/vnodeCache.h @@ -44,11 +44,11 @@ typedef struct { typedef struct { int vnode; char ** pMem; - long freeSlot; + int64_t freeSlot; pthread_mutex_t vmutex; uint64_t count; // kind of transcation ID - long notFreeSlots; - long threshold; + int64_t notFreeSlots; + int64_t threshold; char commitInProcess; int cacheBlockSize; int cacheNumOfBlocks; diff --git a/src/system/inc/vnodeDataFilterFunc.h b/src/system/detail/inc/vnodeDataFilterFunc.h similarity index 100% rename from src/system/inc/vnodeDataFilterFunc.h rename to src/system/detail/inc/vnodeDataFilterFunc.h diff --git a/src/system/inc/vnodeFile.h b/src/system/detail/inc/vnodeFile.h similarity index 85% rename from src/system/inc/vnodeFile.h rename to src/system/detail/inc/vnodeFile.h index c2fada82f8..3202f80ed6 100644 --- a/src/system/inc/vnodeFile.h +++ b/src/system/detail/inc/vnodeFile.h @@ -36,8 +36,9 @@ typedef struct { int64_t sum; int64_t max; int64_t min; - int64_t wsum; - char reserved[16]; + int16_t maxIndex; + int16_t minIndex; + char reserved[20]; } SField; typedef struct { @@ -68,19 +69,19 @@ typedef struct { } SCompInfo; typedef struct { - long tempHeadOffset; - long compInfoOffset; - long oldCompBlockOffset; + int64_t tempHeadOffset; + int64_t compInfoOffset; + int64_t oldCompBlockOffset; - long oldNumOfBlocks; - long newNumOfBlocks; - long finalNumOfBlocks; + int64_t oldNumOfBlocks; + int64_t newNumOfBlocks; + int64_t finalNumOfBlocks; - long oldCompBlockLen; - long newCompBlockLen; - long finalCompBlockLen; + int64_t oldCompBlockLen; + int64_t newCompBlockLen; + int64_t finalCompBlockLen; - long committedPoints; + int64_t committedPoints; int commitSlot; int32_t last : 1; int32_t changed : 1; diff --git a/src/system/inc/vnodeMgmt.h b/src/system/detail/inc/vnodeMgmt.h similarity index 72% rename from src/system/inc/vnodeMgmt.h rename to src/system/detail/inc/vnodeMgmt.h index 0fec3f2437..453795e0bd 100644 --- a/src/system/inc/vnodeMgmt.h +++ b/src/system/detail/inc/vnodeMgmt.h @@ -20,8 +20,16 @@ extern "C" { #endif -int vnodeProcessCreateMeterRequest(char *pMsg); -int vnodeProcessRemoveMeterRequest(char *pMsg); +typedef struct { + char id[20]; + char sid; + void *thandle; + int mgmtIndex; + char status; // 0:offline, 1:online +} SMgmtObj; + +int vnodeProcessCreateMeterRequest(char *pMsg, int msgLen, SMgmtObj *pMgmtObj); +int vnodeProcessRemoveMeterRequest(char *pMsg, int msgLen, SMgmtObj *pMgmtObj); #ifdef __cplusplus } diff --git a/src/system/detail/inc/vnodePeer.h b/src/system/detail/inc/vnodePeer.h new file mode 100644 index 0000000000..d44143e619 --- /dev/null +++ b/src/system/detail/inc/vnodePeer.h @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_VNODEPEER_H +#define TDENGINE_VNODEPEER_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define TSDB_VMSG_SYNC_DATA 1 +#define TSDB_VMSG_FORWARD 2 +#define TSDB_VMSG_SYNC_REQ 3 +#define TSDB_VMSG_SYNC_RSP 4 +#define TSDB_VMSG_SYNC_MUST 5 +#define TSDB_VMSG_STATUS 6 + +#pragma pack(push, 1) + +typedef struct { + char type; + char version; + short sourceVid; + short destVid; +} SFirstPkt; + +typedef struct { + uint64_t lastCreate; + uint64_t lastRemove; + uint32_t fileId; + uint64_t fmagic[]; +} SSyncMsg; + +typedef struct { + char status; + uint64_t version; +} SPeerState; + +typedef struct { + char status : 6; + char ack : 2; + char commitInProcess; + int32_t fileId; // ID for corrupted file, 0 means no corrupted file + uint64_t version; + SPeerState peerStates[]; +} SPeerStatus; + +#pragma pack(pop) + +typedef struct _thread_obj { + pthread_t thread; + int threadId; + int pollFd; + int numOfFds; +} SThreadObj; + +typedef struct { + int numOfThreads; + SThreadObj **pThread; + pthread_t thread; + int threadId; +} SThreadPool; + +typedef struct _vnodePeer { + void * signature; + int ownId; // own vnode ID + uint32_t ip; + char ipstr[20]; // ip string + int vid; + int status; + int syncStatus; + int32_t fileId; // 0 means no corrupted file + uint64_t version; + int commitInProcess; + int syncFd; + int peerFd; // forward FD + void * hbTimer; + void * syncTimer; + SThreadObj *pThread; +} SVnodePeer; + +typedef struct { + SVnodePeer *pVPeer; + uint64_t lastCreate; + uint64_t lastRemove; + uint32_t fileId; + uint64_t fmagic[]; +} SSyncCmd; + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_VNODEPEER_H diff --git a/src/system/inc/vnodeQueryImpl.h b/src/system/detail/inc/vnodeQueryImpl.h similarity index 79% rename from src/system/inc/vnodeQueryImpl.h rename to src/system/detail/inc/vnodeQueryImpl.h index 61e7ff750b..c00af3b8e9 100644 --- a/src/system/inc/vnodeQueryImpl.h +++ b/src/system/detail/inc/vnodeQueryImpl.h @@ -28,7 +28,7 @@ extern "C" { #define GET_QINFO_ADDR(x) ((char*)(x)-offsetof(SQInfo, query)) #define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0) -#define DEFAULT_INTERN_BUF_SIZE 8192L +#define DEFAULT_INTERN_BUF_SIZE 16384L #define INIT_ALLOCATE_DISK_PAGES 60L #define DEFAULT_DATA_FILE_MAPPING_PAGES 2L #define DEFAULT_DATA_FILE_MMAP_WINDOW_SIZE (DEFAULT_DATA_FILE_MAPPING_PAGES * DEFAULT_INTERN_BUF_SIZE) @@ -124,6 +124,7 @@ bool isFixedOutputQuery(SQuery* pQuery); bool isPointInterpoQuery(SQuery* pQuery); bool isTopBottomQuery(SQuery* pQuery); bool isFirstLastRowQuery(SQuery* pQuery); +bool isTSCompQuery(SQuery* pQuery); bool needSupplementaryScan(SQuery* pQuery); bool onDemandLoadDatablock(SQuery* pQuery, int16_t queryRangeSet); @@ -134,7 +135,6 @@ bool doRevisedResultsByLimit(SQInfo* pQInfo); void truncateResultByLimit(SQInfo* pQInfo, int64_t* final, int32_t* interpo); void initCtxOutputBuf(SQueryRuntimeEnv* pRuntimeEnv); -void cleanCtxOutputBuf(SQueryRuntimeEnv* pRuntimeEnv); void resetCtxOutputBuf(SQueryRuntimeEnv* pRuntimeEnv); void forwardCtxOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, int64_t output); @@ -162,7 +162,7 @@ void pointInterpSupporterSetData(SQInfo* pQInfo, SPointInterpoSupporter* pPointI int64_t loadRequiredBlockIntoMem(SQueryRuntimeEnv* pRuntimeEnv, SPositionInfo* position); void doCloseAllOpenedResults(SMeterQuerySupportObj* pSupporter); -void disableFunctForSuppleScanAndSetSortOrder(SQueryRuntimeEnv* pRuntimeEnv, int32_t order); +void disableFunctForSuppleScan(SQueryRuntimeEnv* pRuntimeEnv, int32_t order); void enableFunctForMasterScan(SQueryRuntimeEnv* pRuntimeEnv, int32_t order); int32_t mergeMetersResultToOneGroups(SMeterQuerySupportObj* pSupporter); @@ -184,22 +184,83 @@ int32_t createDataBlocksInfoEx(SMeterDataInfo** pMeterDataInfo, int32_t numOfMet int32_t* nAllocBlocksInfoSize, int64_t addr); void freeMeterBlockInfoEx(SMeterDataBlockInfoEx* pDataBlockInfoEx, int32_t len); -void setExecutionContext(SMeterQuerySupportObj* pSupporter, SOutputRes* outputRes, int32_t meterIdx, int32_t groupIdx); +void setExecutionContext(SMeterQuerySupportObj* pSupporter, SOutputRes* outputRes, int32_t meterIdx, int32_t groupIdx, + SMeterQueryInfo* sqinfo); void setIntervalQueryExecutionContext(SMeterQuerySupportObj* pSupporter, int32_t meterIdx, SMeterQueryInfo* sqinfo); int64_t getQueryStartPositionInCache(SQueryRuntimeEnv* pRuntimeEnv, int32_t* slot, int32_t* pos, bool ignoreQueryRange); int64_t getNextAccessedKeyInData(SQuery* pQuery, int64_t* pPrimaryCol, SBlockInfo* pBlockInfo, int32_t blockStatus); -void setIntervalQueryRange(SMeterQuerySupportObj* pSupporter, int64_t key, SMeterDataInfo* pInfoEx); -void saveIntervalQueryRange(SQuery* pQuery, SMeterQueryInfo* pInfo); -void restoreIntervalQueryRange(SQuery* pQuery, SMeterQueryInfo* pInfo); - uint32_t getDataBlocksForMeters(SMeterQuerySupportObj* pSupporter, SQuery* pQuery, char* pHeaderData, int32_t numOfMeters, SQueryFileInfo* pQueryFileInfo, SMeterDataInfo** pMeterDataInfo); int32_t LoadDatablockOnDemand(SCompBlock* pBlock, SField** pFields, int8_t* blkStatus, SQueryRuntimeEnv* pRuntimeEnv, int32_t fileIdx, int32_t slotIdx, __block_search_fn_t searchFn, bool onDemand); -void setMeterQueryInfo(SMeterQuerySupportObj* pSupporter, SMeterDataInfo* pMeterDataInfo); +/** + * Create SMeterQueryInfo. + * The MeterQueryInfo is created one for each table during super table query + * + * @param skey + * @param ekey + * @return + */ +SMeterQueryInfo* createMeterQueryInfo(SQuery* pQuery, TSKEY skey, TSKEY ekey); + +/** + * Destroy meter query info + * @param pMeterQInfo + * @param numOfCols + */ +void destroyMeterQueryInfo(SMeterQueryInfo *pMeterQueryInfo, int32_t numOfCols); + +/** + * change the meter query info for supplement scan + * @param pMeterQueryInfo + * @param skey + * @param ekey + */ +void changeMeterQueryInfoForSuppleQuery(SMeterQueryInfo *pMeterQueryInfo, TSKEY skey, TSKEY ekey); + +/** + * add the new allocated disk page to meter query info + * the new allocated disk page is used to keep the intermediate (interval) results + * + * @param pMeterQueryInfo + * @param pSupporter + */ +tFilePage* addDataPageForMeterQueryInfo(SMeterQueryInfo *pMeterQueryInfo, SMeterQuerySupportObj *pSupporter); + +/** + * save the query range data into SMeterQueryInfo + * @param pRuntimeEnv + * @param pMeterQueryInfo + */ +void saveIntervalQueryRange(SQueryRuntimeEnv* pRuntimeEnv, SMeterQueryInfo* pMeterQueryInfo); + +/** + * restore the query range data from SMeterQueryInfo to runtime environment + * + * @param pRuntimeEnv + * @param pMeterQueryInfo + */ +void restoreIntervalQueryRange(SQueryRuntimeEnv* pRuntimeEnv, SMeterQueryInfo* pMeterQueryInfo); + +/** + * set the interval query range for the interval query, when handling a data(cache) block + * + * @param pMeterQueryInfo + * @param pSupporter + * @param key + */ +void setIntervalQueryRange(SMeterQueryInfo *pMeterQueryInfo, SMeterQuerySupportObj* pSupporter, int64_t key); + +/** + * set the meter data information + * @param pMeterDataInfo + * @param pMeterObj current query meter object + * @param meterIdx meter index in the sid list + * @param groupId group index, which the meter is belonged to + */ void setMeterDataInfo(SMeterDataInfo* pMeterDataInfo, SMeterObj* pMeterObj, int32_t meterIdx, int32_t groupId); void vnodeSetTagValueInParam(tSidSet* pSidSet, SQueryRuntimeEnv* pRuntimeEnv, SMeterSidExtInfo* pMeterInfo); @@ -210,6 +271,8 @@ void displayInterResult(SData** pdata, SQuery* pQuery, int32_t numOfRows); void vnodePrintQueryStatistics(SMeterQuerySupportObj* pSupporter); +void clearGroupResultBuf(SOutputRes* pOneOutputRes, int32_t nOutputCols); + #ifdef __cplusplus } #endif diff --git a/src/system/inc/vnodeRead.h b/src/system/detail/inc/vnodeRead.h similarity index 76% rename from src/system/inc/vnodeRead.h rename to src/system/detail/inc/vnodeRead.h index 31b4512800..27a785ad85 100644 --- a/src/system/inc/vnodeRead.h +++ b/src/system/detail/inc/vnodeRead.h @@ -60,20 +60,18 @@ typedef struct SQueryFileInfo { int32_t headerFd; /* file handler */ char* pHeaderFileData; /* mmap header files */ size_t headFileSize; - int32_t dataFd; char* pDataFileData; size_t dataFileSize; uint64_t dtFileMappingOffset; int32_t lastFd; - char* pLastFileData; size_t lastFileSize; uint64_t lastFileMappingOffset; } SQueryFileInfo; -typedef struct SQueryCostStatistics { +typedef struct SQueryCostSummary { double cacheTimeUs; double fileTimeUs; @@ -98,44 +96,39 @@ typedef struct SQueryCostStatistics { int64_t totalCompInfoSize; // total comp block size double loadCompInfoUs; // total elapsed time to read comp block info - int64_t tmpBufferInDisk; // size of buffer for intermeidate result -} SQueryCostStatistics; + int64_t tmpBufferInDisk; // size of buffer for intermediate result +} SQueryCostSummary; + +typedef struct SOutputRes { + uint16_t numOfRows; + int32_t nAlloc; + tFilePage** result; + SResultInfo* resultInfo; +} SOutputRes; typedef struct RuntimeEnvironment { SPositionInfo startPos; /* the start position, used for secondary/third iteration */ - SPositionInfo endPos; /* the last access position in query, served as the - start pos of reversed order query */ + SPositionInfo endPos; /* the last access position in query, served as the start pos of reversed order query */ SPositionInfo nextPos; /* start position of the next scan */ - - SData* colDataBuffer[TSDB_MAX_COLUMNS]; - - /* - * for data that requires second/third scan of all data, to denote the column - * need to perform operation refactor to SQLFunctionCtx - */ - bool* go; + SData* colDataBuffer[TSDB_MAX_COLUMNS]; + SResultInfo* resultInfo; // Indicate if data block is loaded, the block is first/last/internal block - int8_t blockStatus; - int32_t internalBufSize; - - SData* primaryColBuffer; - char* unzipBuffer; - char* secondaryUnzipBuffer; - - SQuery* pQuery; - SMeterObj* pMeterObj; - SQLFunctionCtx* pCtx; - - char* buffer; /* column data load buffer, colDataBuffer is point to this value - */ + int8_t blockStatus; + int32_t unzipBufSize; + SData* primaryColBuffer; + char* unzipBuffer; + char* secondaryUnzipBuffer; + SQuery* pQuery; + SMeterObj* pMeterObj; + SQLFunctionCtx* pCtx; + char* buffer; /* column data load buffer, colDataBuffer is point to this value */ SQueryLoadBlockInfo loadBlockInfo; /* record current block load information */ SQueryLoadCompBlockInfo loadCompBlockInfo; /* record current compblock information in SQuery */ /* * header files info, avoid to iterate the directory, the data is acquired - * during - * in query preparation function + * during in query preparation function */ SQueryFileInfo* pHeaderFiles; uint32_t numOfFiles; /* number of files of one vnode during query execution */ @@ -146,33 +139,31 @@ typedef struct RuntimeEnvironment { int16_t scanFlag; /* denotes reversed scan of data or not */ SInterpolationInfo interpoInfo; SData** pInterpoBuf; + SOutputRes* pResult; // reference to SQuerySupporter->pResult + void* hashList; + int32_t usedIndex; // assigned SOutputRes in list - SQueryCostStatistics summary; + STSBuf* pTSBuf; + STSCursor cur; + SQueryCostSummary summary; } SQueryRuntimeEnv; -typedef struct SOutputRes { - uint16_t numOfRows; - int32_t nAlloc; - tFilePage** result; -} SOutputRes; - /* intermediate result during multimeter query involves interval */ typedef struct SMeterQueryInfo { - int64_t lastKey; - int64_t skey; - int64_t ekey; - int32_t numOfRes; - uint32_t numOfPages; - uint32_t numOfAlloc; - - int32_t reverseIndex; // reversed output indicator, start from (numOfRes-1) - int16_t reverseFillRes; // denote if reverse fill the results in - // supplementary scan required or not - int16_t queryRangeSet; // denote if the query range is set, only available - // for interval query - int16_t lastResRows; // - - uint32_t* pageList; + int64_t lastKey; + int64_t skey; + int64_t ekey; + int32_t numOfRes; + uint32_t numOfPages; + uint32_t numOfAlloc; + int32_t reverseIndex; // reversed output indicator, start from (numOfRes-1) + int16_t reverseFillRes; // denote if reverse fill the results in supplementary scan required or not + int16_t queryRangeSet; // denote if the query range is set, only available for interval query + int16_t lastResRows; + int64_t tag; + STSCursor cur; + SResultInfo* resultInfo; + uint32_t* pageList; } SMeterQueryInfo; typedef struct SMeterDataInfo { @@ -202,12 +193,10 @@ typedef struct SMeterQuerySupportObj { */ SOutputRes* pResult; SQueryRuntimeEnv runtimeEnv; - - int64_t rawSKey; - int64_t rawEKey; - - int32_t subgroupIdx; - int32_t offset; /* offset in group result set of subgroup */ + int64_t rawSKey; + int64_t rawEKey; + int32_t subgroupIdx; + int32_t offset; /* offset in group result set of subgroup */ tSidSet* pSidSet; @@ -231,6 +220,9 @@ typedef struct SMeterQuerySupportObj { SMeterDataInfo* pMeterDataInfo; + TSKEY* tsList; + int32_t tsNum; + } SMeterQuerySupportObj; typedef struct _qinfo { @@ -246,14 +238,13 @@ typedef struct _qinfo { int killed; struct _qinfo *prev, *next; - SQuery query; - int num; - int totalPoints; - int pointsRead; - int pointsReturned; - int pointsInterpo; - int code; - + SQuery query; + int num; + int totalPoints; + int pointsRead; + int pointsReturned; + int pointsInterpo; + int code; char bufIndex; char changed; char over; @@ -266,13 +257,14 @@ typedef struct _qinfo { } SQInfo; -int32_t vnodeQuerySingleMeterPrepare(SQInfo* pQInfo, SMeterObj* pMeterObj, SMeterQuerySupportObj* pSMultiMeterObj); +int32_t vnodeQuerySingleMeterPrepare(SQInfo* pQInfo, SMeterObj* pMeterObj, SMeterQuerySupportObj* pSMultiMeterObj, + void* param); void vnodeQueryFreeQInfoEx(SQInfo* pQInfo); bool vnodeParametersSafetyCheck(SQuery* pQuery); -int32_t vnodeMultiMeterQueryPrepare(SQInfo* pQInfo, SQuery* pQuery); +int32_t vnodeMultiMeterQueryPrepare(SQInfo* pQInfo, SQuery* pQuery, void* param); /** * decrease the numofQuery of each table that is queried, enable the diff --git a/src/system/inc/vnodeShell.h b/src/system/detail/inc/vnodeShell.h similarity index 100% rename from src/system/inc/vnodeShell.h rename to src/system/detail/inc/vnodeShell.h diff --git a/src/system/inc/vnodeStore.h b/src/system/detail/inc/vnodeStore.h similarity index 88% rename from src/system/inc/vnodeStore.h rename to src/system/detail/inc/vnodeStore.h index e38abad347..d4eedd4ce0 100644 --- a/src/system/inc/vnodeStore.h +++ b/src/system/detail/inc/vnodeStore.h @@ -16,13 +16,20 @@ #ifndef TDENGINE_VNODESTORE_H #define TDENGINE_VNODESTORE_H +#include + #ifdef __cplusplus extern "C" { #endif void vnodeProcessDataFromVnode(SIntMsg *msg, void *tcpHandle); + void vnodeCalcOpenVnodes(); +bool vnodeRemoveDataFileFromLinkFile(char* linkFile, char* de_name); + +int vnodeInitInfo(); + #ifdef __cplusplus } #endif diff --git a/src/system/inc/vnodeSystem.h b/src/system/detail/inc/vnodeSystem.h similarity index 80% rename from src/system/inc/vnodeSystem.h rename to src/system/detail/inc/vnodeSystem.h index 10745e7a72..e69b0b9f1e 100644 --- a/src/system/inc/vnodeSystem.h +++ b/src/system/detail/inc/vnodeSystem.h @@ -16,12 +16,26 @@ #ifndef TDENGINE_VNODESYSTEM_H #define TDENGINE_VNODESYSTEM_H +#include + #ifdef __cplusplus extern "C" { #endif +void vnodeCleanUpSystem(); + +void vnodePrintSystemInfo(); + int vnodeInitSystem(); +int vnodeCfgDynamicOptions(char *msg); + +int vnodeInitStore(); + +int vnodeInitPeer(int numOfThreads); + +int vnodeInitMgmt(); + #ifdef __cplusplus } #endif diff --git a/src/system/inc/vnodeTagMgmt.h b/src/system/detail/inc/vnodeTagMgmt.h similarity index 96% rename from src/system/inc/vnodeTagMgmt.h rename to src/system/detail/inc/vnodeTagMgmt.h index 898a2d3944..320ef56453 100644 --- a/src/system/inc/vnodeTagMgmt.h +++ b/src/system/detail/inc/vnodeTagMgmt.h @@ -52,7 +52,7 @@ typedef struct tSidSet { typedef int32_t (*__ext_compar_fn_t)(const void *p1, const void *p2, void *param); tSidSet *tSidSetCreate(struct SMeterSidExtInfo **pMeterSidExtInfo, int32_t numOfMeters, SSchema *pSchema, - int32_t numOfTags, int16_t *orderList, int32_t numOfOrderCols); + int32_t numOfTags, SColIndexEx *colList, int32_t numOfOrderCols); tTagSchema *tCreateTagSchema(SSchema *pSchema, int32_t numOfTagCols); diff --git a/src/system/inc/vnodeUtil.h b/src/system/detail/inc/vnodeUtil.h similarity index 94% rename from src/system/inc/vnodeUtil.h rename to src/system/detail/inc/vnodeUtil.h index b7efcfaa38..8705794660 100644 --- a/src/system/inc/vnodeUtil.h +++ b/src/system/detail/inc/vnodeUtil.h @@ -29,6 +29,8 @@ extern "C" { #define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) #define EXTRA_BYTES 2 // for possible compression deflation +#define GET_COL_DATA_POS(query, index, step) ((query)->pos + (index)*(step)) + int vnodeGetEid(int days); int vnodeCheckFileIntegrity(FILE *fp); @@ -80,6 +82,8 @@ void vnodeClearMeterState(SMeterObj* pMeterObj, int32_t state); bool vnodeIsMeterState(SMeterObj* pMeterObj, int32_t state); void vnodeSetMeterDeleting(SMeterObj* pMeterObj); bool vnodeIsSafeToDeleteMeter(SVnodeObj* pVnode, int32_t sid); +void vnodeFreeColumnInfo(SColumnInfo* pColumnInfo); +bool isGroupbyNormalCol(SSqlGroupbyExpr* pExpr); #ifdef __cplusplus } diff --git a/src/system/src/dnodeMgmt.c b/src/system/detail/src/dnodeMgmt.c similarity index 66% rename from src/system/src/dnodeMgmt.c rename to src/system/detail/src/dnodeMgmt.c index 28e723f3f0..3a6c9e50a8 100644 --- a/src/system/src/dnodeMgmt.c +++ b/src/system/detail/src/dnodeMgmt.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include #include @@ -27,66 +28,65 @@ #include "vnodeSystem.h" #include "vnodeUtil.h" -int vnodeProcessVPeersMsg(char *msg); -int vnodeProcessCreateMeterMsg(char *pMsg); -int vnodeProcessFreeVnodeRequest(char *pMsg); -int vnodeProcessVPeerCfgRsp(char *msg); -int vnodeProcessMeterCfgRsp(char *msg); -int vnodeProcessAlterStreamRequest(char *pMsg); - -void mgmtProcessMsgFromVnode(SSchedMsg *sched); -void *mgmtQhandle; - -int vnodeSendMsgToMgmt(char *msg) { - SSchedMsg schedMsg; - schedMsg.fp = mgmtProcessMsgFromVnode; - schedMsg.msg = msg; - schedMsg.ahandle = NULL; - schedMsg.thandle = NULL; - taosScheduleTask(mgmtQhandle, &schedMsg); - - return 0; -} - -void vnodeProcessMsgFromMgmt(SSchedMsg *sched) { - char msgType = *sched->msg; - char *content = sched->msg + 1; - - dTrace("msg:%s is received from mgmt", taosMsg[msgType]); - +SMgmtObj mgmtObj; +extern uint64_t tsCreatedTime; + +int vnodeProcessVPeersMsg(char *msg, int msgLen, SMgmtObj *pMgmtObj); +int vnodeProcessCreateMeterMsg(char *pMsg, int msgLen); +int vnodeProcessFreeVnodeRequest(char *pMsg, int msgLen, SMgmtObj *pMgmtObj); +int vnodeProcessVPeerCfgRsp(char *msg, int msgLen, SMgmtObj *pMgmtObj); +int vnodeProcessMeterCfgRsp(char *msg, int msgLen, SMgmtObj *pMgmtObj); +int vnodeProcessCfgDnodeRequest(char *cont, int contLen, SMgmtObj *pMgmtObj); +int vnodeProcessAlterStreamRequest(char *pMsg, int msgLen, SMgmtObj *pObj); +void vnodeUpdateHeadFile(int vnode, int oldTables, int newTables); +void vnodeOpenVnode(int vnode); +void vnodeCleanUpOneVnode(int vnode); + +int vnodeSaveCreateMsgIntoQueue(SVnodeObj *pVnode, char *pMsg, int msgLen); + +char *taosBuildRspMsgToMnodeWithSize(SMgmtObj *pObj, char type, int size); +char *taosBuildReqMsgToMnodeWithSize(SMgmtObj *pObj, char type, int size); +char *taosBuildRspMsgToMnode(SMgmtObj *pObj, char type); +char *taosBuildReqMsgToMnode(SMgmtObj *pObj, char type); +int taosSendSimpleRspToMnode(SMgmtObj *pObj, char rsptype, char code); +int taosSendMsgToMnode(SMgmtObj *pObj, char *msg, int msgLen); + +void vnodeProcessMsgFromMgmt(char *content, int msgLen, int msgType, SMgmtObj *pObj) { if (msgType == TSDB_MSG_TYPE_CREATE) { - vnodeProcessCreateMeterRequest(content); + vnodeProcessCreateMeterRequest(content, msgLen, pObj); } else if (msgType == TSDB_MSG_TYPE_VPEERS) { - vnodeProcessVPeersMsg(content); + vnodeProcessVPeersMsg(content, msgLen, pObj); } else if (msgType == TSDB_MSG_TYPE_VPEER_CFG_RSP) { - vnodeProcessVPeerCfgRsp(content); + vnodeProcessVPeerCfgRsp(content, msgLen, pObj); } else if (msgType == TSDB_MSG_TYPE_METER_CFG_RSP) { - vnodeProcessMeterCfgRsp(content); + vnodeProcessMeterCfgRsp(content, msgLen, pObj); } else if (msgType == TSDB_MSG_TYPE_REMOVE) { - vnodeProcessRemoveMeterRequest(content); + vnodeProcessRemoveMeterRequest(content, msgLen, pObj); } else if (msgType == TSDB_MSG_TYPE_FREE_VNODE) { - vnodeProcessFreeVnodeRequest(content); + vnodeProcessFreeVnodeRequest(content, msgLen, pObj); + } else if (msgType == TSDB_MSG_TYPE_CFG_PNODE) { + vnodeProcessCfgDnodeRequest(content, msgLen, pObj); } else if (msgType == TSDB_MSG_TYPE_ALTER_STREAM) { - vnodeProcessAlterStreamRequest(content); + vnodeProcessAlterStreamRequest(content, msgLen, pObj); + } else if (msgType == TSDB_MSG_TYPE_GRANT_RSP) { + // do nothing } else { dError("%s is not processed", taosMsg[msgType]); } - - free(sched->msg); } -int vnodeProcessMeterCfgRsp(char *pMsg) { +int vnodeProcessMeterCfgRsp(char *pMsg, int msgLen, SMgmtObj *pObj) { int code = *pMsg; if (code == 0) { - vnodeProcessCreateMeterMsg(pMsg + 1); + vnodeProcessCreateMeterMsg(pMsg + 1, msgLen - 1); } else { STaosRsp *pRsp; pRsp = (STaosRsp *)pMsg; int32_t *pint = (int32_t *)pRsp->more; int vnode = htonl(*pint); int sid = htonl(*(pint + 1)); - dError("vid:%d, sid:%d, meter is not configured, remove it", vnode, sid); + dError("vid:%d, sid:%d, code:%d, meter is not configured, remove it", vnode, sid, code); int ret = vnodeRemoveMeterObj(vnode, sid); dTrace("vid:%d, sid:%d, meter delete ret:%d", vnode, sid, ret); } @@ -94,12 +94,11 @@ int vnodeProcessMeterCfgRsp(char *pMsg) { return 0; } -int vnodeProcessCreateMeterRequest(char *pMsg) { +int vnodeProcessCreateMeterRequest(char *pMsg, int msgLen, SMgmtObj *pObj) { SCreateMsg *pCreate; int code = 0; int vid; SVnodeObj * pVnode; - char * pStart; pCreate = (SCreateMsg *)pMsg; vid = htons(pCreate->vnode); @@ -117,28 +116,24 @@ int vnodeProcessCreateMeterRequest(char *pMsg) { goto _over; } - code = vnodeProcessCreateMeterMsg(pMsg); + if (pVnode->syncStatus == TSDB_SSTATUS_SYNCING) { + code = vnodeSaveCreateMsgIntoQueue(pVnode, pMsg, msgLen); + dTrace("vid:%d, create msg is saved into sync queue", vid); + } else { + code = vnodeProcessCreateMeterMsg(pMsg, msgLen); + } _over: - - pStart = (char *)malloc(128); - if (pStart == NULL) return 0; - - *pStart = TSDB_MSG_TYPE_CREATE_RSP; - pMsg = pStart + 1; - - *pMsg = code; - vnodeSendMsgToMgmt(pStart); + taosSendSimpleRspToMnode(pObj, TSDB_MSG_TYPE_CREATE_RSP, code); return code; } -int vnodeProcessAlterStreamRequest(char *pMsg) { +int vnodeProcessAlterStreamRequest(char *pMsg, int msgLen, SMgmtObj *pObj) { SAlterStreamMsg *pAlter; int code = 0; int vid, sid; SVnodeObj * pVnode; - char * pStart; pAlter = (SAlterStreamMsg *)pMsg; vid = htons(pAlter->vnode); @@ -182,19 +177,12 @@ int vnodeProcessAlterStreamRequest(char *pMsg) { vnodeSaveMeterObjToFile(pMeterObj); _over: - pStart = (char *)malloc(128); - if (pStart == NULL) return 0; - - *pStart = TSDB_MSG_TYPE_ALTER_STREAM_RSP; - pMsg = pStart + 1; - - *pMsg = code; - vnodeSendMsgToMgmt(pStart); + taosSendSimpleRspToMnode(pObj, TSDB_MSG_TYPE_ALTER_STREAM_RSP, code); return code; } -int vnodeProcessCreateMeterMsg(char *pMsg) { +int vnodeProcessCreateMeterMsg(char *pMsg, int msgLen) { int code; SMeterObj * pObj = NULL; SConnSec connSec; @@ -296,11 +284,10 @@ _create_over: return code; } -int vnodeProcessRemoveMeterRequest(char *pMsg) { +int vnodeProcessRemoveMeterRequest(char *pMsg, int msgLen, SMgmtObj *pMgmtObj) { SMeterObj * pObj; SRemoveMeterMsg *pRemove; int code = 0; - char * pStart; pRemove = (SRemoveMeterMsg *)pMsg; pRemove->vnode = htons(pRemove->vnode); @@ -330,22 +317,13 @@ int vnodeProcessRemoveMeterRequest(char *pMsg) { dTrace("vid:%d sid:%d id:%s, meterObj is removed", pRemove->vnode, pRemove->sid, pRemove->meterId); _remove_over: - - pStart = (char *)malloc(128); - if (pStart == NULL) return 0; - - *pStart = TSDB_MSG_TYPE_REMOVE_RSP; - pMsg = pStart + 1; - - *pMsg = code; - vnodeSendMsgToMgmt(pStart); - + taosSendSimpleRspToMnode(pMgmtObj, TSDB_MSG_TYPE_REMOVE_RSP, code); return 0; } -int vnodeProcessVPeerCfg(char *msg) { +int vnodeProcessVPeerCfg(char *msg, int msgLen, SMgmtObj *pMgmtObj) { SVPeersMsg *pMsg = (SVPeersMsg *)msg; - int vnode; + int i, vnode; vnode = htonl(pMsg->vnode); if (vnode >= TSDB_MAX_VNODES) { @@ -371,15 +349,35 @@ int vnodeProcessVPeerCfg(char *msg) { pCfg->blocksPerMeter = htons(pCfg->blocksPerMeter); pCfg->rowsInFileBlock = htonl(pCfg->rowsInFileBlock); - dTrace("vid:%d, vgroup:%d, vpeer cfg received, sessions:%d, current session:%d", vnode, pCfg->vgId, pCfg->maxSessions, - vnodeList[vnode].cfg.maxSessions); + if (pCfg->replications > 0) { + dTrace("vid:%d, vpeer cfg received, replica:%d session:%d, vnodeList replica:%d session:%d", + vnode, pCfg->replications, pCfg->maxSessions, vnodeList[vnode].cfg.replications, vnodeList[vnode].cfg.maxSessions); + for (i = 0; i < pCfg->replications; ++i) { + pMsg->vpeerDesc[i].vnode = htonl(pMsg->vpeerDesc[i].vnode); + pMsg->vpeerDesc[i].ip = htonl(pMsg->vpeerDesc[i].ip); + dTrace("vid:%d, vpeer:%d ip:0x%x vid:%d ", vnode, i, pMsg->vpeerDesc[i].ip, pMsg->vpeerDesc[i].vnode); + } + } if (vnodeList[vnode].cfg.maxSessions == 0) { if (pCfg->maxSessions > 0) { return vnodeCreateVnode(vnode, pCfg, pMsg->vpeerDesc); } } else { - if (pCfg->maxSessions <= 0) { + if (pCfg->maxSessions > 0) { + if (pCfg->maxSessions != vnodeList[vnode].cfg.maxSessions) { + vnodeCleanUpOneVnode(vnode); + } + + vnodeConfigVPeers(vnode, pCfg->replications, pMsg->vpeerDesc); + vnodeSaveVnodeCfg(vnode, pCfg, pMsg->vpeerDesc); + + if (pCfg->maxSessions != vnodeList[vnode].cfg.maxSessions) { + vnodeUpdateHeadFile(vnode, vnodeList[vnode].cfg.maxSessions, pCfg->maxSessions); + vnodeList[vnode].cfg.maxSessions = pCfg->maxSessions; + vnodeOpenVnode(vnode); + } + } else { vnodeRemoveVnode(vnode); } } @@ -387,13 +385,13 @@ int vnodeProcessVPeerCfg(char *msg) { return 0; } -int vnodeProcessVPeerCfgRsp(char *msg) { +int vnodeProcessVPeerCfgRsp(char *msg, int msgLen, SMgmtObj *pMgmtObj) { STaosRsp *pRsp; pRsp = (STaosRsp *)msg; if (pRsp->code == 0) { - vnodeProcessVPeerCfg(pRsp->more); + vnodeProcessVPeerCfg(pRsp->more, msgLen - sizeof(STaosRsp), pMgmtObj); } else { int32_t *pint = (int32_t *)pRsp->more; int vnode = htonl(*pint); @@ -408,33 +406,30 @@ int vnodeProcessVPeerCfgRsp(char *msg) { return 0; } -int vnodeProcessVPeersMsg(char *msg) { - int code = 0; - char *pStart, *pMsg; +int vnodeProcessVPeersMsg(char *msg, int msgLen, SMgmtObj *pMgmtObj) { + int code = 0; - code = vnodeProcessVPeerCfg(msg); + code = vnodeProcessVPeerCfg(msg, msgLen, pMgmtObj); + char * pStart; STaosRsp * pRsp; SVPeersMsg *pVPeersMsg = (SVPeersMsg *)msg; - pStart = (char *)malloc(128); - if (pStart == NULL) return 0; - - *pStart = TSDB_MSG_TYPE_VPEERS_RSP; - pMsg = pStart + 1; + pStart = taosBuildRspMsgToMnode(pMgmtObj, TSDB_MSG_TYPE_VPEERS_RSP); + if (pStart == NULL) return -1; - pRsp = (STaosRsp *)pMsg; + pRsp = (STaosRsp *)pStart; pRsp->code = code; memcpy(pRsp->more, pVPeersMsg->cfg.db, TSDB_DB_NAME_LEN); - vnodeSendMsgToMgmt(pStart); + msgLen = sizeof(STaosRsp) + TSDB_DB_NAME_LEN; + taosSendMsgToMnode(pMgmtObj, pStart, msgLen); return code; } -int vnodeProcessFreeVnodeRequest(char *pMsg) { +int vnodeProcessFreeVnodeRequest(char *pMsg, int msgLen, SMgmtObj *pMgmtObj) { SFreeVnodeMsg *pFree; - char * pStart; pFree = (SFreeVnodeMsg *)pMsg; pFree->vnode = htons(pFree->vnode); @@ -448,53 +443,53 @@ int vnodeProcessFreeVnodeRequest(char *pMsg) { int32_t code = vnodeRemoveVnode(pFree->vnode); assert(code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS); - pStart = (char *)malloc(128); - if (pStart == NULL) return 0; + taosSendSimpleRspToMnode(pMgmtObj, TSDB_MSG_TYPE_FREE_VNODE_RSP, code); + return 0; +} - *pStart = TSDB_MSG_TYPE_FREE_VNODE_RSP; - pMsg = pStart + 1; +int vnodeProcessCfgDnodeRequest(char *cont, int contLen, SMgmtObj *pMgmtObj) { + SCfgMsg *pCfg = (SCfgMsg *)cont; - *pMsg = code; - vnodeSendMsgToMgmt(pStart); + int code = tsCfgDynamicOptions(pCfg->config); + + taosSendSimpleRspToMnode(pMgmtObj, TSDB_MSG_TYPE_CFG_PNODE_RSP, code); return 0; } -int vnodeSendVpeerCfgMsg(int vnode) { +void vnodeSendVpeerCfgMsg(int vnode) { + char * pMsg, *pStart; + int msgLen; SVpeerCfgMsg *pCfg; - char * pStart, *pMsg; - - pStart = (char *)malloc(256); - if (pStart == NULL) return -1; + SMgmtObj * pObj = &mgmtObj; - *pStart = TSDB_MSG_TYPE_VPEER_CFG; - pMsg = pStart + 1; + pStart = taosBuildReqMsgToMnode(pObj, TSDB_MSG_TYPE_VPEER_CFG); + if (pStart == NULL) return; + pMsg = pStart; pCfg = (SVpeerCfgMsg *)pMsg; pCfg->vnode = htonl(vnode); pMsg += sizeof(SVpeerCfgMsg); - vnodeSendMsgToMgmt(pStart); - - return 0; + msgLen = pMsg - pStart; + taosSendMsgToMnode(pObj, pStart, msgLen); } int vnodeSendMeterCfgMsg(int vnode, int sid) { + char * pMsg, *pStart; + int msgLen; SMeterCfgMsg *pCfg; - char * pStart, *pMsg; + SMgmtObj * pObj = &mgmtObj; - pStart = (char *)malloc(256); - if (pStart == NULL) return 0; - - *pStart = TSDB_MSG_TYPE_METER_CFG; - pMsg = pStart + 1; + pStart = taosBuildReqMsgToMnode(pObj, TSDB_MSG_TYPE_METER_CFG); + if (pStart == NULL) return -1; + pMsg = pStart; pCfg = (SMeterCfgMsg *)pMsg; pCfg->vnode = htonl(vnode); pCfg->sid = htonl(sid); pMsg += sizeof(SMeterCfgMsg); - vnodeSendMsgToMgmt(pStart); - - return 0; + msgLen = pMsg - pStart; + return taosSendMsgToMnode(pObj, pStart, msgLen); } diff --git a/src/system/src/dnodeService.c b/src/system/detail/src/dnodeService.c similarity index 96% rename from src/system/src/dnodeService.c rename to src/system/detail/src/dnodeService.c index 17c4c736d1..86d3b4a795 100644 --- a/src/system/src/dnodeService.c +++ b/src/system/detail/src/dnodeService.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include #include @@ -70,6 +71,8 @@ int main(int argc, char *argv[]) { printf("gitinfo: %s\n", gitinfo); printf("buildinfo: %s\n", buildinfo); return 0; + } else if (strcmp(argv[i], "-k") == 0) { + dnodeParseParameterK(); } } diff --git a/src/system/src/dnodeSystem.c b/src/system/detail/src/dnodeSystem.c similarity index 63% rename from src/system/src/dnodeSystem.c rename to src/system/detail/src/dnodeSystem.c index 2fed8556c1..a0fdf95fc8 100644 --- a/src/system/src/dnodeSystem.c +++ b/src/system/detail/src/dnodeSystem.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include #include @@ -29,16 +30,30 @@ #include "tglobalcfg.h" #include "vnode.h" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Woverflow" + SModule tsModule[TSDB_MOD_MAX]; uint32_t tsModuleStatus; pthread_mutex_t dmutex; extern int vnodeSelectReqNum; extern int vnodeInsertReqNum; +void * tsStatusTimer = NULL; bool tsDnodeStopping = false; +int dnodeCheckConfig(); void dnodeCountRequest(SCountInfo *info); void dnodeInitModules() { + tsModule[TSDB_MOD_MGMT].name = "mgmt"; + tsModule[TSDB_MOD_MGMT].initFp = mgmtInitSystem; + tsModule[TSDB_MOD_MGMT].cleanUpFp = mgmtCleanUpSystem; + tsModule[TSDB_MOD_MGMT].startFp = mgmtStartSystem; + tsModule[TSDB_MOD_MGMT].stopFp = mgmtStopSystem; + tsModule[TSDB_MOD_MGMT].num = tsNumOfMPeers; + tsModule[TSDB_MOD_MGMT].curNum = 0; + tsModule[TSDB_MOD_MGMT].equalVnodeNum = tsMgmtEqualVnodeNum; + tsModule[TSDB_MOD_HTTP].name = "http"; tsModule[TSDB_MOD_HTTP].initFp = httpInitSystem; tsModule[TSDB_MOD_HTTP].cleanUpFp = httpCleanUpSystem; @@ -62,25 +77,28 @@ void dnodeCleanUpSystem() { if (tsDnodeStopping) return; tsDnodeStopping = true; - for (int mod = 0; mod < TSDB_MOD_MAX; ++mod) { - if (tsModule[mod].num != 0 && tsModule[mod].stopFp) (*tsModule[mod].stopFp)(); - if (tsModule[mod].num != 0 && tsModule[mod].cleanUpFp) (*tsModule[mod].cleanUpFp)(); + if (tsStatusTimer != NULL) { + taosTmrStopA(&tsStatusTimer); + tsStatusTimer = NULL; } - mgmtCleanUpSystem(); - vnodeCleanUpVnodes(); - - taosCloseLogger(); -} + for (int mod = 1; mod < TSDB_MOD_MAX; ++mod) { + if (tsModule[mod].num != 0 && tsModule[mod].stopFp) { + (*tsModule[mod].stopFp)(); + } + if (tsModule[mod].num != 0 && tsModule[mod].cleanUpFp) { + (*tsModule[mod].cleanUpFp)(); + } + } -void taosCreateTierDirectory() { - char fileName[128]; + if (tsModule[TSDB_MOD_MGMT].num != 0 && tsModule[TSDB_MOD_MGMT].cleanUpFp) { + (*tsModule[TSDB_MOD_MGMT].cleanUpFp)(); + } - sprintf(fileName, "%s/tsdb", tsDirectory); - mkdir(fileName, 0755); + vnodeCleanUpVnodes(); - sprintf(fileName, "%s/data", tsDirectory); - mkdir(fileName, 0755); + taosCloseLogger(); + taosCleanupTier(); } void dnodeCheckDbRunning(const char* dir) { @@ -117,17 +135,11 @@ int dnodeInitSystem() { return -1; } - strcpy(tsDirectory, dataDir); - if (stat(dataDir, &dirstat) < 0) { - mkdir(dataDir, 0755); + if (taosCreateTierDirectory() != 0) { + dError("TDengine init tier directory failed"); + return -1; } - taosCreateTierDirectory(); - - sprintf(mgmtDirectory, "%s/mgmt", tsDirectory); - sprintf(tsDirectory, "%s/tsdb", dataDir); - dnodeCheckDbRunning(dataDir); - tsPrintGlobalConfig(); dPrint("Server IP address is:%s", tsInternalIp); @@ -136,8 +148,13 @@ int dnodeInitSystem() { dnodeInitModules(); pthread_mutex_init(&dmutex, NULL); - dPrint("starting to initialize TDengine engine ..."); + dPrint("starting to initialize TDengine ..."); + vnodeInitQHandle(); + if (dnodeInitSystemSpec() < 0) { + return -1; + } + for (int mod = 0; mod < TSDB_MOD_MAX; ++mod) { if (tsModule[mod].num != 0 && tsModule[mod].initFp) { if ((*tsModule[mod].initFp)() != 0) { @@ -152,29 +169,55 @@ int dnodeInitSystem() { return -1; } - if (mgmtInitSystem() != 0) { - dError("TDengine mgmt initialization failed"); - return -1; - } - monitorCountReqFp = dnodeCountRequest; - for (int mod = 0; mod < TSDB_MOD_MAX; ++mod) { - if (tsModule[mod].num != 0 && tsModule[mod].startFp) { - if ((*tsModule[mod].startFp)() != 0) { - dError("failed to start TDengine module:%d", mod); - return -1; - } - } - } + dnodeStartModuleSpec(); dPrint("TDengine is initialized successfully"); return 0; } +void dnodeProcessModuleStatus(uint32_t status) { + if (tsDnodeStopping) return; + + int news = status; + int olds = tsModuleStatus; + + for (int moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) { + int newStatus = news & (1 << moduleType); + int oldStatus = olds & (1 << moduleType); + + if (oldStatus > 0) { + if (newStatus == 0) { + if (tsModule[moduleType].stopFp) { + dPrint("module:%s is stopped on this node", tsModule[moduleType].name); + (*tsModule[moduleType].stopFp)(); + } + } + } else if (oldStatus == 0) { + if (newStatus > 0) { + if (tsModule[moduleType].startFp) { + dPrint("module:%s is started on this node", tsModule[moduleType].name); + (*tsModule[moduleType].startFp)(); + } + } + } else { + } + } + tsModuleStatus = status; +} + +void dnodeResetSystem() { + dPrint("reset the system ..."); + for (int vnode = 0; vnode < TSDB_MAX_VNODES; ++vnode) vnodeRemoveVnode(vnode); + mgmtStopSystem(); +} + void dnodeCountRequest(SCountInfo *info) { httpGetReqCount(&info->httpReqNum); info->selectReqNum = __sync_fetch_and_and(&vnodeSelectReqNum, 0); info->insertReqNum = __sync_fetch_and_and(&vnodeInsertReqNum, 0); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/src/system/src/mgmtAcct.c b/src/system/detail/src/mgmtAcct.c similarity index 91% rename from src/system/src/mgmtAcct.c rename to src/system/detail/src/mgmtAcct.c index 6ad2b3a461..dac67518e9 100644 --- a/src/system/src/mgmtAcct.c +++ b/src/system/detail/src/mgmtAcct.c @@ -13,12 +13,14 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include "mgmt.h" #include "tschemautil.h" -SAcctObj acctObj; +int mgmtGetAcctsNum(); +SShowObj *mgmtGetNextAcct(SShowObj *pShow, SAcctObj **pAcct); int mgmtAddDbIntoAcct(SAcctObj *pAcct, SDbObj *pDb) { pthread_mutex_lock(&pAcct->mutex); @@ -124,13 +126,3 @@ int mgmtRemoveConnFromAcct(SConnObj *pConn) { return 0; } - -void mgmtCheckAcct() { - SAcctObj *pAcct = &acctObj; - pAcct->acctId = 0; - strcpy(pAcct->user, "root"); - - mgmtCreateUser(pAcct, "root", "taosdata"); - mgmtCreateUser(pAcct, "monitor", tsInternalPass); - mgmtCreateUser(pAcct, "_root", tsInternalPass); -} diff --git a/src/system/src/mgmtConn.c b/src/system/detail/src/mgmtConn.c similarity index 99% rename from src/system/src/mgmtConn.c rename to src/system/detail/src/mgmtConn.c index c5621bd343..b3fb24de6b 100644 --- a/src/system/src/mgmtConn.c +++ b/src/system/detail/src/mgmtConn.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "mgmt.h" #include #include "taosmsg.h" diff --git a/src/system/src/mgmtDb.c b/src/system/detail/src/mgmtDb.c similarity index 82% rename from src/system/src/mgmtDb.c rename to src/system/detail/src/mgmtDb.c index a2c409be9c..ae4c5bed7c 100644 --- a/src/system/src/mgmtDb.c +++ b/src/system/detail/src/mgmtDb.c @@ -13,8 +13,10 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "mgmt.h" #include +#include "mgmtBalance.h" #include "tschemautil.h" void *dbSdb = NULL; @@ -32,6 +34,9 @@ void *mgmtDbActionAfterBatchUpdate(void *row, char *str, int size, int *ssize); void *mgmtDbActionReset(void *row, char *str, int size, int *ssize); void *mgmtDbActionDestroy(void *row, char *str, int size, int *ssize); +int mgmtCheckDbGrant(); +int mgmtCheckDbLimit(SAcctObj *pAcct); + void mgmtDbActionInit() { mgmtDbActionFp[SDB_TYPE_INSERT] = mgmtDbActionInsert; mgmtDbActionFp[SDB_TYPE_DELETE] = mgmtDbActionDelete; @@ -64,8 +69,9 @@ void mgmtGetAcctStr(char *src, char *dest) { } int mgmtInitDbs() { - void * pNode = NULL; - SDbObj *pDb = NULL; + void * pNode = NULL; + SDbObj * pDb = NULL; + SAcctObj *pAcct = NULL; mgmtDbActionInit(); @@ -89,7 +95,12 @@ int mgmtInitDbs() { pDb->vgStatus = TSDB_VG_STATUS_READY; pDb->vgTimer = NULL; pDb->pMetric = NULL; - mgmtAddDbIntoAcct(&acctObj, pDb); + pAcct = mgmtGetAcct(pDb->cfg.acct); + if (pAcct != NULL) + mgmtAddDbIntoAcct(pAcct, pDb); + else { + mError("db:%s acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct); + } } SDbObj tObj; @@ -123,19 +134,26 @@ int mgmtCheckDbParams(SCreateDbMsg *pCreate) { if (pCreate->commitTime < 0) pCreate->commitTime = tsCommitTime; // if (pCreate->compression < 0) pCreate->compression = tsCompression; // if (pCreate->commitLog < 0) pCreate->commitLog = tsCommitLog; - if (pCreate->replications < 0) pCreate->replications = 1; // + if (pCreate->replications < 0) pCreate->replications = tsReplications; // if (pCreate->rowsInFileBlock < 0) pCreate->rowsInFileBlock = tsRowsInFileBlock; // if (pCreate->cacheNumOfBlocks.fraction < 0) pCreate->cacheNumOfBlocks.fraction = tsAverageCacheBlocks; // + //-1 for balance - if (pCreate->replications != 1) { - mTrace("invalid db option replications: %d", pCreate->replications); - return TSDB_CODE_INVALID_OPTION; - } +#ifdef CLUSTER + if (pCreate->replications > TSDB_VNODES_SUPPORT - 1) pCreate->replications = TSDB_VNODES_SUPPORT - 1; +#else + pCreate->replications = 1; +#endif if (pCreate->commitLog < 0 || pCreate->commitLog > 1) { mTrace("invalid db option commitLog: %d", pCreate->commitLog); return TSDB_CODE_INVALID_OPTION; } + + if (pCreate->replications < TSDB_REPLICA_MIN_NUM || pCreate->replications > TSDB_REPLICA_MAX_NUM) { + mTrace("invalid db option replications: %d", pCreate->replications); + return TSDB_CODE_INVALID_OPTION; + } if (pCreate->daysPerFile < TSDB_FILE_MIN_PARTITION_RANGE || pCreate->daysPerFile > TSDB_FILE_MAX_PARTITION_RANGE) { mTrace("invalid db option daysPerFile: %d valid range: %d--%d", pCreate->daysPerFile, TSDB_FILE_MIN_PARTITION_RANGE, @@ -199,7 +217,7 @@ int mgmtCheckDbParams(SCreateDbMsg *pCreate) { return TSDB_CODE_INVALID_OPTION; } - if (pCreate->blocksPerMeter < 0) pCreate->blocksPerMeter = tsNumOfBlocksPerMeter; + if (pCreate->blocksPerMeter < 0) pCreate->blocksPerMeter = pCreate->cacheNumOfBlocks.totalBlocks / 4; if (pCreate->blocksPerMeter > pCreate->cacheNumOfBlocks.totalBlocks * 3 / 4) { pCreate->blocksPerMeter = pCreate->cacheNumOfBlocks.totalBlocks * 3 / 4; } @@ -212,11 +230,11 @@ int mgmtCheckDbParams(SCreateDbMsg *pCreate) { int mgmtCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate) { SDbObj *pDb; + int code; - int numOfDbs = sdbGetNumOfRows(dbSdb); - if (numOfDbs >= tsMaxDbs) { - mWarn("numOfDbs:%d, exceed tsMaxDbs:%d", numOfDbs, tsMaxDbs); - return TSDB_CODE_TOO_MANY_DATABSES; + code = mgmtCheckDbLimit(pAcct); + if (code != 0) { + return code; } pDb = (SDbObj *)sdbGetRow(dbSdb, pCreate->db); @@ -224,11 +242,16 @@ int mgmtCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate) { return TSDB_CODE_DB_ALREADY_EXIST; } - int code = mgmtCheckDbParams(pCreate); + code = mgmtCheckDbParams(pCreate); if (code != TSDB_CODE_SUCCESS) return code; assert(pCreate->daysToKeep1 <= pCreate->daysToKeep2 && pCreate->daysToKeep2 <= pCreate->daysToKeep); + code = mgmtCheckDbGrant(); + if (code != 0) { + return code; + } + pDb = malloc(sizeof(SDbObj)); memset(pDb, 0, sizeof(SDbObj)); strcpy(pDb->name, pCreate->db); @@ -251,15 +274,23 @@ int mgmtSetDbDropping(SDbObj *pDb) { SVgObj *pVgroup = pDb->pHead; while (pVgroup != NULL) { - SDnodeObj *pDnode = &dnodeObj; - if (pDnode == NULL) continue; - - SVnodeLoad *pVload = &pDnode->vload[pVgroup->vnodeGid[0].vnode]; - if (pVload->dropStatus != TSDB_VN_STATUS_DROPPING) { - pVload->dropStatus = TSDB_VN_STATUS_DROPPING; - mPrint("vnode:%d db:%s set to dropping status", pVgroup->vnodeGid[0].vnode, pDb->name); + for (int i = 0; i < pVgroup->numOfVnodes; i++) { + SVnodeGid *pVnodeGid = pVgroup->vnodeGid + i; + SDnodeObj *pDnode = mgmtGetDnode(pVnodeGid->ip); + if (pDnode == NULL) continue; + + SVnodeLoad *pVload = &pDnode->vload[pVnodeGid->vnode]; + if (pVload->dropStatus != TSDB_VN_STATUS_DROPPING) { + pVload->dropStatus = TSDB_VN_STATUS_DROPPING; + + mPrint("dnode:%s vnode:%d db:%s set to dropping status", taosIpStr(pDnode->privateIp), pVnodeGid->vnode, pDb->name); + if (mgmtUpdateDnode(pDnode) < 0) { + mError("db:%s drop failed, dnode sdb update error", pDb->name); + return TSDB_CODE_SDB_ERROR; + } + } } - mgmtSendFreeVnodeMsg(pVgroup->vnodeGid[0].vnode); + mgmtSendFreeVnodeMsg(pVgroup); pVgroup = pVgroup->next; } @@ -278,16 +309,19 @@ int mgmtSetDbDropping(SDbObj *pDb) { bool mgmtCheckDropDbFinished(SDbObj *pDb) { SVgObj *pVgroup = pDb->pHead; while (pVgroup) { - SDnodeObj *pDnode = &dnodeObj; + for (int i = 0; i < pVgroup->numOfVnodes; i++) { + SVnodeGid *pVnodeGid = pVgroup->vnodeGid + i; + SDnodeObj *pDnode = mgmtGetDnode(pVnodeGid->ip); - if (pDnode->status == TSDB_STATUS_OFFLINE) continue; + if (pDnode == NULL) continue; + if (pDnode->status == TSDB_STATUS_OFFLINE) continue; - SVnodeLoad *pVload = &pDnode->vload[pVgroup->vnodeGid[0].vnode]; - if (pVload->dropStatus == TSDB_VN_STATUS_DROPPING) { - mTrace("dnode:0x%x vnode:%d db:%s wait dropping", pDnode->privateIp, pVgroup->vnodeGid[0].vnode, pDb->name); - return false; + SVnodeLoad *pVload = &pDnode->vload[pVnodeGid->vnode]; + if (pVload->dropStatus == TSDB_VN_STATUS_DROPPING) { + mTrace("dnode:%s, vnode:%d db:%s wait dropping", taosIpStr(pDnode->privateIp), pVnodeGid->vnode, pDb->name); + return false; + } } - pVgroup = pVgroup->next; } @@ -316,10 +350,7 @@ int mgmtDropDb(SDbObj *pDb) { if (!finished) { SVgObj *pVgroup = pDb->pHead; while (pVgroup != NULL) { - SDnodeObj *pDnode = &dnodeObj; - if (pDnode == NULL) continue; - SVnodeLoad *pVload = &pDnode->vload[pVgroup->vnodeGid[0].vnode]; - mgmtSendFreeVnodeMsg(pVgroup->vnodeGid[0].vnode); + mgmtSendFreeVnodeMsg(pVgroup); pVgroup = pVgroup->next; } return TSDB_CODE_ACTION_IN_PROGRESS; @@ -373,10 +404,34 @@ int mgmtAlterDb(SAcctObj *pAcct, SAlterDbMsg *pAlter) { return TSDB_CODE_INVALID_DB; } + int oldReplicaNum = pDb->cfg.replications; if (pAlter->daysToKeep > 0) { mTrace("db:%s daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, pAlter->daysToKeep); pDb->cfg.daysToKeep = pAlter->daysToKeep; + } else if (pAlter->replications > 0) { + mTrace("db:%s replica:%d change to %d", pDb->name, pDb->cfg.replications, pAlter->replications); + if (pAlter->replications < TSDB_REPLICA_MIN_NUM || pAlter->replications > TSDB_REPLICA_MAX_NUM) { + mError("invalid db option replica: %d valid range: %d--%d", pAlter->replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); + return TSDB_CODE_INVALID_OPTION; + } + pDb->cfg.replications = pAlter->replications; + } else if (pAlter->maxSessions > 0) { + mTrace("db:%s tables:%d change to %d", pDb->name, pDb->cfg.maxSessions, pAlter->maxSessions); + if (pAlter->maxSessions < TSDB_MIN_TABLES_PER_VNODE || pAlter->maxSessions > TSDB_MAX_TABLES_PER_VNODE) { + mError("invalid db option tables: %d valid range: %d--%d", pAlter->maxSessions, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); + return TSDB_CODE_INVALID_OPTION; + } + if (pAlter->maxSessions < pDb->cfg.maxSessions) { + mError("invalid db option tables: %d should larger than original:%d", pAlter->maxSessions, pDb->cfg.maxSessions); + return TSDB_CODE_INVALID_OPTION; + } + return TSDB_CODE_INVALID_OPTION; + //The modification of tables needs to rewrite the head file, so disable this option + //pDb->cfg.maxSessions = pAlter->maxSessions; } else { + mError("db:%s alter msg, replica:%d, keep:%d, tables:%d, origin replica:%d keep:%d", pDb->name, + pAlter->replications, pAlter->maxSessions, pAlter->daysToKeep, + pDb->cfg.replications, pDb->cfg.daysToKeep); return TSDB_CODE_INVALID_OPTION; } @@ -384,6 +439,24 @@ int mgmtAlterDb(SAcctObj *pAcct, SAlterDbMsg *pAlter) { return TSDB_CODE_SDB_ERROR; } + SVgObj *pVgroup = pDb->pHead; + while (pVgroup != NULL) { + mgmtUpdateVgroupState(pVgroup, LB_VGROUP_STATE_UPDATE, 0); + if (oldReplicaNum < pDb->cfg.replications) { + if (!mgmtAddVnode(pVgroup, NULL, NULL)) { + mWarn("db:%s vgroup:%d not enough dnode to add vnode", pAlter->db, pVgroup->vgId); + code = TSDB_CODE_NO_ENOUGH_DNODES; + } + } + if (pAlter->maxSessions > 0) { + //rebuild meterList in mgmtVgroup.c + sdbUpdateRow(vgSdb, pVgroup, tsVgUpdateSize, 0); + } + mgmtSendVPeersMsg(pVgroup); + pVgroup = pVgroup->next; + } + mgmtStartBalanceTimer(10); + return code; } @@ -391,6 +464,7 @@ int mgmtUseDb(SConnObj *pConn, char *name) { SDbObj *pDb; int code = TSDB_CODE_INVALID_DB; + // here change the default db for connect. pDb = mgmtGetDb(name); if (pDb) { pConn->pDb = pDb; @@ -708,7 +782,11 @@ int mgmtRetrieveDbs(SShowObj *pShow, char *data, int rows, SConnObj *pConn) { cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; +#ifdef _TD_ARM_32_ + *(int32_t *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions - 1)); +#else *(float *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions - 1)); +#endif cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -748,7 +826,8 @@ int mgmtRetrieveDbs(SShowObj *pShow, char *data, int rows, SConnObj *pConn) { } void *mgmtDbActionInsert(void *row, char *str, int size, int *ssize) { - SDbObj *pDb = (SDbObj *)row; + SDbObj * pDb = (SDbObj *)row; + SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); pDb->pHead = NULL; pDb->pTail = NULL; @@ -756,13 +835,13 @@ void *mgmtDbActionInsert(void *row, char *str, int size, int *ssize) { pDb->numOfTables = 0; pDb->vgTimer = NULL; pDb->pMetric = NULL; - mgmtAddDbIntoAcct(&acctObj, pDb); + mgmtAddDbIntoAcct(pAcct, pDb); return NULL; } void *mgmtDbActionDelete(void *row, char *str, int size, int *ssize) { SDbObj * pDb = (SDbObj *)row; - SAcctObj *pAcct = &acctObj; + SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); mgmtRemoveDbFromAcct(pAcct, pDb); return NULL; diff --git a/src/system/detail/src/mgmtDnode.c b/src/system/detail/src/mgmtDnode.c new file mode 100644 index 0000000000..36e3a41595 --- /dev/null +++ b/src/system/detail/src/mgmtDnode.c @@ -0,0 +1,391 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE + +#include +#include +#include + +#include "dnodeSystem.h" +#include "mgmt.h" +#include "tschemautil.h" +#include "tstatus.h" + +bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int moduleType); +int mgmtGetDnodesNum(); +void*mgmtGetNextDnode(SShowObj *pShow, SDnodeObj **pDnode); +bool mgmtCheckConfigShow(SGlobalConfig *cfg); + +void mgmtSetDnodeMaxVnodes(SDnodeObj *pDnode) { + int maxVnodes = pDnode->numOfCores * tsNumOfVnodesPerCore; + maxVnodes = maxVnodes > TSDB_MAX_VNODES ? TSDB_MAX_VNODES : maxVnodes; + maxVnodes = maxVnodes < TSDB_MIN_VNODES ? TSDB_MIN_VNODES : maxVnodes; + if (pDnode->numOfTotalVnodes != 0) { + maxVnodes = pDnode->numOfTotalVnodes; + } + if (pDnode->alternativeRole == TSDB_DNODE_ROLE_MGMT) { + maxVnodes = 0; + } + + pDnode->numOfVnodes = maxVnodes; + pDnode->numOfFreeVnodes = maxVnodes; + pDnode->openVnodes = 0; + +#ifdef CLUSTER + pDnode->status = TSDB_STATUS_OFFLINE; +#else + pDnode->status = TSDB_STATUS_READY; +#endif +} + +void mgmtCalcNumOfFreeVnodes(SDnodeObj *pDnode) { + int totalVnodes = 0; + + for (int i = 0; i < pDnode->numOfVnodes; ++i) { + SVnodeLoad *pVload = pDnode->vload + i; + if (pVload->vgId != 0) { + mTrace("dnode:%s, calc free vnodes, exist vnode:%d, vgroup:%d, state:%d %s, dropstate:%d %s, syncstatus:%d %s", + taosIpStr(pDnode->privateIp), i, pVload->vgId, + pVload->status, sdbDnodeStatusStr[pVload->status], + pVload->dropStatus, sdbVnodeDropStateStr[pVload->dropStatus], + pVload->syncStatus, sdbVnodeSyncStatusStr[pVload->syncStatus]); + totalVnodes++; + } + } + + pDnode->numOfFreeVnodes = pDnode->numOfVnodes - totalVnodes; + mTrace("dnode:%s, calc free vnodes, numOfVnodes:%d, numOfFreeVnodes:%d, totalVnodes:%d", + taosIpStr(pDnode->privateIp), pDnode->numOfVnodes, pDnode->numOfFreeVnodes, totalVnodes); +} + +void mgmtSetDnodeVgid(SVnodeGid vnodeGid[], int numOfVnodes, int vgId) { + SDnodeObj *pDnode; + + for (int i = 0; i < numOfVnodes; ++i) { + pDnode = mgmtGetDnode(vnodeGid[i].ip); + if (pDnode) { + SVnodeLoad *pVload = pDnode->vload + vnodeGid[i].vnode; + memset(pVload, 0, sizeof(SVnodeLoad)); + pVload->vnode = vnodeGid[i].vnode; + pVload->vgId = vgId; + mTrace("dnode:%s, vnode:%d add to vgroup:%d", taosIpStr(vnodeGid[i].ip), vnodeGid[i].vnode, pVload->vgId); + mgmtCalcNumOfFreeVnodes(pDnode); + } else { + mError("dnode:%s, not in dnode DB!!!", taosIpStr(vnodeGid[i].ip)); + } + } +} + +void mgmtUnSetDnodeVgid(SVnodeGid vnodeGid[], int numOfVnodes) { + SDnodeObj *pDnode; + + for (int i = 0; i < numOfVnodes; ++i) { + pDnode = mgmtGetDnode(vnodeGid[i].ip); + if (pDnode) { + SVnodeLoad *pVload = pDnode->vload + vnodeGid[i].vnode; + mTrace("dnode:%s, vnode:%d remove from vgroup:%d", taosIpStr(vnodeGid[i].ip), vnodeGid[i].vnode, pVload->vgId); + memset(pVload, 0, sizeof(SVnodeLoad)); + mgmtCalcNumOfFreeVnodes(pDnode); + } else { + mError("dnode:%s not in dnode DB!!!", taosIpStr(vnodeGid[i].ip)); + } + } +} + +int mgmtGetDnodeMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { + int cols = 0; + + if (strcmp(pConn->pAcct->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; + + SSchema *pSchema = tsGetSchema(pMeta); + + pShow->bytes[cols] = 16; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "IP"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "created time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "open vnodes"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "free vnodes"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 10; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "status"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 18; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "balance state"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 16; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "public ip"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + + pShow->numOfRows = mgmtGetDnodesNum(); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + pShow->pNode = NULL; + + return 0; +} + +int mgmtRetrieveDnodes(SShowObj *pShow, char *data, int rows, SConnObj *pConn) { + int numOfRows = 0; + SDnodeObj *pDnode = NULL; + char * pWrite; + int cols = 0; + char ipstr[20]; + + while (numOfRows < rows) { + pShow->pNode = mgmtGetNextDnode(pShow, (SDnodeObj **)&pDnode); + if (pDnode == NULL) break; + + cols = 0; + + tinet_ntoa(ipstr, pDnode->privateIp); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, ipstr); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pDnode->createdTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDnode->openVnodes; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDnode->numOfFreeVnodes; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, sdbDnodeStatusStr[pDnode->status]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, sdbDnodeBalanceStateStr[pDnode->lbState]); + cols++; + + tinet_ntoa(ipstr, pDnode->publicIp); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, ipstr); + cols++; + + numOfRows++; + } + + pShow->numOfReads += numOfRows; + return numOfRows; +} + +int mgmtGetModuleMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { + int cols = 0; + + if (strcmp(pConn->pAcct->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; + + SSchema *pSchema = tsGetSchema(pMeta); + + pShow->bytes[cols] = 16; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "IP"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 10; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "module type"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 10; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "module status"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + + pShow->numOfRows = 0; + SDnodeObj *pDnode = NULL; + while (1) { + pShow->pNode = mgmtGetNextDnode(pShow, (SDnodeObj **)&pDnode); + if (pDnode == NULL) break; + for (int moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) { + if (mgmtCheckModuleInDnode(pDnode, moduleType)) { + pShow->numOfRows++; + } + } + } + + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + pShow->pNode = NULL; + + return 0; +} + +int mgmtRetrieveModules(SShowObj *pShow, char *data, int rows, SConnObj *pConn) { + int numOfRows = 0; + SDnodeObj *pDnode = NULL; + char * pWrite; + int cols = 0; + char ipstr[20]; + + while (numOfRows < rows) { + pShow->pNode = mgmtGetNextDnode(pShow, (SDnodeObj **)&pDnode); + if (pDnode == NULL) break; + + for (int moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) { + if (!mgmtCheckModuleInDnode(pDnode, moduleType)) { + continue; + } + + cols = 0; + + tinet_ntoa(ipstr, pDnode->privateIp); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, ipstr); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, tsModule[moduleType].name); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, sdbDnodeStatusStr[pDnode->status]); + cols++; + + numOfRows++; + } + } + + pShow->numOfReads += numOfRows; + return numOfRows; +} + +int mgmtGetConfigMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { + int cols = 0; + + if (strcmp(pConn->pAcct->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; + + SSchema *pSchema = tsGetSchema(pMeta); + + pShow->bytes[cols] = TSDB_CFG_OPTION_LEN; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "config name"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_CFG_VALUE_LEN; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "config value"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + + pShow->numOfRows = 0; + for (int i = tsGlobalConfigNum - 1; i >= 0; --i) { + SGlobalConfig *cfg = tsGlobalConfig + i; + if (!mgmtCheckConfigShow(cfg)) continue; + pShow->numOfRows++; + } + + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + pShow->pNode = NULL; + + return 0; +} + +int mgmtRetrieveConfigs(SShowObj *pShow, char *data, int rows, SConnObj *pConn) { + int numOfRows = 0; + + for (int i = tsGlobalConfigNum - 1; i >= 0 && numOfRows < rows; --i) { + SGlobalConfig *cfg = tsGlobalConfig + i; + if (!mgmtCheckConfigShow(cfg)) continue; + + char *pWrite; + int cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + snprintf(pWrite, TSDB_CFG_OPTION_LEN, "%s", cfg->option); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + switch (cfg->valType) { + case TSDB_CFG_VTYPE_SHORT: + snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr)); + numOfRows++; + break; + case TSDB_CFG_VTYPE_INT: + snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int32_t *)cfg->ptr)); + numOfRows++; + break; + case TSDB_CFG_VTYPE_UINT: + snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((uint32_t *)cfg->ptr)); + numOfRows++; + break; + case TSDB_CFG_VTYPE_FLOAT: + snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%f", *((float *)cfg->ptr)); + numOfRows++; + break; + case TSDB_CFG_VTYPE_STRING: + case TSDB_CFG_VTYPE_IPSTR: + case TSDB_CFG_VTYPE_DIRECTORY: + snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%s", (char *)cfg->ptr); + numOfRows++; + break; + default: + break; + } + } + + pShow->numOfReads += numOfRows; + return numOfRows; +} diff --git a/src/system/detail/src/mgmtDnodeInt.c b/src/system/detail/src/mgmtDnodeInt.c new file mode 100644 index 0000000000..2b7fe3cf44 --- /dev/null +++ b/src/system/detail/src/mgmtDnodeInt.c @@ -0,0 +1,484 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include +#include + +#include "dnodeSystem.h" +#include "mgmt.h" +#include "mgmtBalance.h" +#include "tutil.h" + +void mgmtProcessMsgFromDnode(char *content, int msgLen, int msgType, SDnodeObj *pObj); +int mgmtSendVPeersMsg(SVgObj *pVgroup); +char *mgmtBuildVpeersIe(char *pMsg, SVgObj *pVgroup, int vnode); +char *mgmtBuildCreateMeterIe(STabObj *pMeter, char *pMsg, int vnode); + +/* + * functions for communicate between dnode and mnode + */ +char *taosBuildRspMsgToDnodeWithSize(SDnodeObj *pObj, char type, int size); +char *taosBuildReqMsgToDnodeWithSize(SDnodeObj *pObj, char type, int size); +char *taosBuildRspMsgToDnode(SDnodeObj *pObj, char type); +char *taosBuildReqMsgToDnode(SDnodeObj *pObj, char type); +int taosSendSimpleRspToDnode(SDnodeObj *pObj, char rsptype, char code); +int taosSendMsgToDnode(SDnodeObj *pObj, char *msg, int msgLen); + +int mgmtProcessMeterCfgMsg(char *cont, int contLen, SDnodeObj *pObj) { + char * pMsg, *pStart; + int msgLen = 0; + STabObj * pMeter = NULL; + SMeterCfgMsg *pCfg = (SMeterCfgMsg *)cont; + SVgObj * pVgroup; + + int vnode = htonl(pCfg->vnode); + int sid = htonl(pCfg->sid); + + pStart = taosBuildRspMsgToDnodeWithSize(pObj, TSDB_MSG_TYPE_METER_CFG_RSP, 64000); + if (pStart == NULL) { + taosSendSimpleRspToDnode(pObj, TSDB_MSG_TYPE_METER_CFG_RSP, TSDB_CODE_SERV_OUT_OF_MEMORY); + return 0; + } + pMsg = pStart; + + if (vnode < pObj->numOfVnodes) { + int vgId = pObj->vload[vnode].vgId; + + pVgroup = mgmtGetVgroup(vgId); + if (pVgroup) pMeter = pVgroup->meterList[sid]; + } + + if (pMeter) { + *pMsg = 0; // code + pMsg++; + pMsg = mgmtBuildCreateMeterIe(pMeter, pMsg, vnode); + } else { + mTrace("dnode:%s, vnode:%d sid:%d, meter not there", taosIpStr(pObj->privateIp), vnode, sid); + *pMsg = TSDB_CODE_INVALID_METER_ID; + pMsg++; + + *(int32_t *)pMsg = htonl(vnode); + pMsg += sizeof(int32_t); + *(int32_t *)pMsg = htonl(sid); + pMsg += sizeof(int32_t); + } + + msgLen = pMsg - pStart; + taosSendMsgToDnode(pObj, pStart, msgLen); + + return 0; +} + +int mgmtProcessVpeerCfgMsg(char *cont, int contLen, SDnodeObj *pObj) { + char * pMsg, *pStart; + int msgLen = 0; + SVpeerCfgMsg *pCfg = (SVpeerCfgMsg *)cont; + SVgObj * pVgroup = NULL; + + int vnode = htonl(pCfg->vnode); + + pStart = taosBuildRspMsgToDnode(pObj, TSDB_MSG_TYPE_VPEER_CFG_RSP); + if (pStart == NULL) return 0; + pMsg = pStart; + + if (vnode < pObj->numOfVnodes) pVgroup = mgmtGetVgroup(pObj->vload[vnode].vgId); + + if (pVgroup) { + *pMsg = 0; + pMsg++; + pMsg = mgmtBuildVpeersIe(pMsg, pVgroup, vnode); + mTrace("dnode:%s, vnode:%d, vgroup:%d, send create meter msg, code:%d", taosIpStr(pObj->privateIp), vnode, pVgroup->vgId, *pMsg); + } else { + mTrace("dnode:%s, vnode:%d, no vgroup info, vgroup:%d", taosIpStr(pObj->privateIp), vnode, pObj->vload[vnode].vgId); + *pMsg = TSDB_CODE_INVALID_VALUE; + pMsg++; + *(int32_t *)pMsg = htonl(vnode); + pMsg += sizeof(int32_t); + } + + msgLen = pMsg - pStart; + taosSendMsgToDnode(pObj, pStart, msgLen); + + return 0; +} + +int mgmtProcessCreateRsp(char *msg, int msgLen, SDnodeObj *pObj) { return 0; } + +int mgmtProcessFreeVnodeRsp(char *msg, int msgLen, SDnodeObj *pObj) { return 0; } + +int mgmtProcessVPeersRsp(char *msg, int msgLen, SDnodeObj *pObj) { + STaosRsp *pRsp = (STaosRsp *)msg; + + SDbObj *pDb = mgmtGetDb(pRsp->more); + if (!pDb) { + mError("dnode:%s, db not find, code:%d", taosIpStr(pObj->privateIp), pRsp->code); + return 0; + } + + if (pDb->vgStatus != TSDB_VG_STATUS_IN_PROGRESS) { + mTrace("dnode:%s, db:%s vpeer rsp already disposed, code:%d", taosIpStr(pObj->privateIp), pRsp->more, pRsp->code); + return 0; + } + + if (pRsp->code == 0) { + pDb->vgStatus = TSDB_VG_STATUS_READY; + mTrace("dnode:%s, db:%s vgroup is created in dnode", taosIpStr(pObj->privateIp), pRsp->more); + return 0; + } + + if (pRsp->code == TSDB_CODE_VG_COMMITLOG_INIT_FAILED) { + pDb->vgStatus = TSDB_VG_STATUS_COMMITLOG_INIT_FAILED; + } else { + pDb->vgStatus = TSDB_VG_STATUS_INIT_FAILED; + } + mError("dnode:%s, db:%s vgroup create failed, code:%d", taosIpStr(pObj->privateIp), pRsp->more, pRsp->code); + + return 0; +} + +void mgmtProcessMsgFromDnode(char *content, int msgLen, int msgType, SDnodeObj *pObj) { + if (msgType == TSDB_MSG_TYPE_METER_CFG) { + mgmtProcessMeterCfgMsg(content, msgLen - sizeof(SIntMsg), pObj); + } else if (msgType == TSDB_MSG_TYPE_VPEER_CFG) { + mgmtProcessVpeerCfgMsg(content, msgLen - sizeof(SIntMsg), pObj); + } else if (msgType == TSDB_MSG_TYPE_CREATE_RSP) { + mgmtProcessCreateRsp(content, msgLen - sizeof(SIntMsg), pObj); + } else if (msgType == TSDB_MSG_TYPE_REMOVE_RSP) { + // do nothing + } else if (msgType == TSDB_MSG_TYPE_VPEERS_RSP) { + mgmtProcessVPeersRsp(content, msgLen - sizeof(SIntMsg), pObj); + } else if (msgType == TSDB_MSG_TYPE_FREE_VNODE_RSP) { + mgmtProcessFreeVnodeRsp(content, msgLen - sizeof(SIntMsg), pObj); + } else if (msgType == TSDB_MSG_TYPE_CFG_PNODE_RSP) { + // do nothing; + } else if (msgType == TSDB_MSG_TYPE_ALTER_STREAM_RSP) { + // do nothing; + } else { + mError("%s from dnode is not processed", taosMsg[msgType]); + } +} + +char *mgmtBuildCreateMeterIe(STabObj *pMeter, char *pMsg, int vnode) { + SCreateMsg *pCreateMeter; + + pCreateMeter = (SCreateMsg *)pMsg; + pCreateMeter->vnode = htons(vnode); + pCreateMeter->sid = htonl(pMeter->gid.sid); + pCreateMeter->uid = pMeter->uid; + memcpy(pCreateMeter->meterId, pMeter->meterId, TSDB_METER_ID_LEN); + + // pCreateMeter->lastCreate = htobe64(pVgroup->lastCreate); + pCreateMeter->timeStamp = htobe64(pMeter->createdTime); + /* + pCreateMeter->spi = pSec->spi; + pCreateMeter->encrypt = pSec->encrypt; + memcpy(pCreateMeter->cipheringKey, pSec->cipheringKey, TSDB_KEY_LEN); + memcpy(pCreateMeter->secret, pSec->secret, TSDB_KEY_LEN); + */ + pCreateMeter->sversion = htonl(pMeter->sversion); + pCreateMeter->numOfColumns = htons(pMeter->numOfColumns); + SSchema *pSchema = mgmtGetMeterSchema(pMeter); + + for (int i = 0; i < pMeter->numOfColumns; ++i) { + pCreateMeter->schema[i].type = pSchema[i].type; + /* strcpy(pCreateMeter->schema[i].name, pSchema[i].name); */ + pCreateMeter->schema[i].bytes = htons(pSchema[i].bytes); + pCreateMeter->schema[i].colId = htons(pSchema[i].colId); + } + + pMsg = ((char *)(pCreateMeter->schema)) + pMeter->numOfColumns * sizeof(SMColumn); + pCreateMeter->sqlLen = 0; + + if (pMeter->pSql) { + int len = strlen(pMeter->pSql) + 1; + pCreateMeter->sqlLen = htons(len); + strcpy(pMsg, pMeter->pSql); + pMsg += len; + } + + return pMsg; +} + +int mgmtSendCreateMsgToVgroup(STabObj *pMeter, SVgObj *pVgroup) { + char * pMsg, *pStart; + int i, msgLen = 0; + SDnodeObj *pObj; + uint64_t timeStamp; + + timeStamp = taosGetTimestampMs(); + + for (i = 0; i < pVgroup->numOfVnodes; ++i) { + //if (pVgroup->vnodeGid[i].ip == 0) continue; + + pObj = mgmtGetDnode(pVgroup->vnodeGid[i].ip); + if (pObj == NULL) continue; + + pStart = taosBuildReqMsgToDnodeWithSize(pObj, TSDB_MSG_TYPE_CREATE, 64000); + if (pStart == NULL) continue; + pMsg = mgmtBuildCreateMeterIe(pMeter, pStart, pVgroup->vnodeGid[i].vnode); + msgLen = pMsg - pStart; + + taosSendMsgToDnode(pObj, pStart, msgLen); + } + + pVgroup->lastCreate = timeStamp; + + return 0; +} + +int mgmtSendRemoveMeterMsgToDnode(STabObj *pMeter, SVgObj *pVgroup) { + SRemoveMeterMsg *pRemove; + char * pMsg, *pStart; + int i, msgLen = 0; + SDnodeObj * pObj; + char ipstr[20]; + uint64_t timeStamp; + + timeStamp = taosGetTimestampMs(); + + for (i = 0; i < pVgroup->numOfVnodes; ++i) { + //if (pVgroup->vnodeGid[i].ip == 0) continue; + + pObj = mgmtGetDnode(pVgroup->vnodeGid[i].ip); + if (pObj == NULL) continue; + + pStart = taosBuildReqMsgToDnode(pObj, TSDB_MSG_TYPE_REMOVE); + if (pStart == NULL) continue; + pMsg = pStart; + + pRemove = (SRemoveMeterMsg *)pMsg; + pRemove->vnode = htons(pVgroup->vnodeGid[i].vnode); + pRemove->sid = htonl(pMeter->gid.sid); + memcpy(pRemove->meterId, pMeter->meterId, TSDB_METER_ID_LEN); + + pMsg += sizeof(SRemoveMeterMsg); + msgLen = pMsg - pStart; + + taosSendMsgToDnode(pObj, pStart, msgLen); + + tinet_ntoa(ipstr, pVgroup->vnodeGid[i].ip); + mTrace("dnode:%s vid:%d, send remove meter msg, sid:%d status:%d", ipstr, pVgroup->vnodeGid[i].vnode, + pMeter->gid.sid, pObj->status); + } + + pVgroup->lastRemove = timeStamp; + + return 0; +} + +int mgmtSendAlterStreamMsgToDnode(STabObj *pMeter, SVgObj *pVgroup) { + SAlterStreamMsg *pAlter; + char * pMsg, *pStart; + int i, msgLen = 0; + SDnodeObj * pObj; + + for (i = 0; i < pVgroup->numOfVnodes; ++i) { + if (pVgroup->vnodeGid[i].ip == 0) continue; + + pObj = mgmtGetDnode(pVgroup->vnodeGid[i].ip); + if (pObj == NULL) continue; + + pStart = taosBuildReqMsgToDnode(pObj, TSDB_MSG_TYPE_ALTER_STREAM); + if (pStart == NULL) continue; + pMsg = pStart; + + pAlter = (SAlterStreamMsg *)pMsg; + pAlter->vnode = htons(pVgroup->vnodeGid[i].vnode); + pAlter->sid = htonl(pMeter->gid.sid); + pAlter->uid = pMeter->uid; + pAlter->status = pMeter->status; + + pMsg += sizeof(SAlterStreamMsg); + msgLen = pMsg - pStart; + + taosSendMsgToDnode(pObj, pStart, msgLen); + } + + return 0; +} + +char *mgmtBuildVpeersIe(char *pMsg, SVgObj *pVgroup, int vnode) { + SVPeersMsg *pVPeers = (SVPeersMsg *)pMsg; + SDbObj * pDb; + + pDb = mgmtGetDb(pVgroup->dbName); + pVPeers->vnode = htonl(vnode); + + pVPeers->cfg = pDb->cfg; + SVnodeCfg *pCfg = &pVPeers->cfg; + pCfg->vgId = htonl(pVgroup->vgId); + pCfg->maxSessions = htonl(pCfg->maxSessions); + pCfg->cacheBlockSize = htonl(pCfg->cacheBlockSize); + pCfg->cacheNumOfBlocks.totalBlocks = htonl(pCfg->cacheNumOfBlocks.totalBlocks); + pCfg->daysPerFile = htonl(pCfg->daysPerFile); + pCfg->daysToKeep1 = htonl(pCfg->daysToKeep1); + pCfg->daysToKeep2 = htonl(pCfg->daysToKeep2); + pCfg->daysToKeep = htonl(pCfg->daysToKeep); + pCfg->commitTime = htonl(pCfg->commitTime); + pCfg->blocksPerMeter = htons(pCfg->blocksPerMeter); + pCfg->replications = (char)pVgroup->numOfVnodes; + pCfg->rowsInFileBlock = htonl(pCfg->rowsInFileBlock); + +#ifdef CLUSTER + SVPeerDesc *vpeerDesc = pVPeers->vpeerDesc; + + pMsg = (char *)(pVPeers->vpeerDesc); + + for (int j = 0; j < pVgroup->numOfVnodes; ++j) { + vpeerDesc[j].ip = htonl(pVgroup->vnodeGid[j].ip); + vpeerDesc[j].vnode = htonl(pVgroup->vnodeGid[j].vnode); + pMsg += sizeof(SVPeerDesc); + } +#endif + + return pMsg; +} + +int mgmtSendVPeersMsg(SVgObj *pVgroup) { + SDnodeObj *pDnode; + char * pMsg, *pStart; + int msgLen = 0; + + for (int i = 0; i < pVgroup->numOfVnodes; ++i) { + pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].ip); + if (pDnode == NULL) { + mError("dnode:%s not there", taosIpStr(pVgroup->vnodeGid[i].ip)); + continue; + } + + pDnode->vload[pVgroup->vnodeGid[i].vnode].vgId = pVgroup->vgId; + mgmtUpdateDnode(pDnode); + + if (pDnode->thandle && pVgroup->numOfVnodes >= 1) { + pStart = taosBuildReqMsgToDnode(pDnode, TSDB_MSG_TYPE_VPEERS); + if (pStart == NULL) continue; + pMsg = mgmtBuildVpeersIe(pStart, pVgroup, pVgroup->vnodeGid[i].vnode); + msgLen = pMsg - pStart; + + taosSendMsgToDnode(pDnode, pStart, msgLen); + } + } + + return 0; +} + +int mgmtSendOneFreeVnodeMsg(SVnodeGid *pVnodeGid) { + SFreeVnodeMsg *pFreeVnode; + char * pMsg, *pStart; + int msgLen = 0; + SDnodeObj * pDnode; + + pDnode = mgmtGetDnode(pVnodeGid->ip); + if (pDnode == NULL) { + mError("dnode:%s not there", taosIpStr(pVnodeGid->ip)); + return -1; + } + + if (pDnode->thandle == NULL) { + mTrace("dnode:%s offline, failed to send Vpeer msg", taosIpStr(pVnodeGid->ip)); + return -1; + } + + pStart = taosBuildReqMsgToDnode(pDnode, TSDB_MSG_TYPE_FREE_VNODE); + if (pStart == NULL) return -1; + pMsg = pStart; + + pFreeVnode = (SFreeVnodeMsg *)pMsg; + pFreeVnode->vnode = htons(pVnodeGid->vnode); + + pMsg += sizeof(SFreeVnodeMsg); + + msgLen = pMsg - pStart; + taosSendMsgToDnode(pDnode, pStart, msgLen); + + return 0; +} + +int mgmtSendFreeVnodeMsg(SVgObj *pVgroup) { + for (int i = 0; i < pVgroup->numOfVnodes; ++i) { + mgmtSendOneFreeVnodeMsg(pVgroup->vnodeGid + i); + } + + return 0; +} + +int mgmtCfgDynamicOptions(SDnodeObj *pDnode, char *msg) { + char *option, *value; + int olen, valen; + + paGetToken(msg, &option, &olen); + if (strncasecmp(option, "unremove", 8) == 0) { + mgmtSetDnodeUnRemove(pDnode); + return TSDB_CODE_SUCCESS; + } else if (strncasecmp(option, "score", 5) == 0) { + paGetToken(option + olen + 1, &value, &valen); + if (valen > 0) { + int score = atoi(value); + mTrace("dnode:%s, custom score set from:%d to:%d", taosIpStr(pDnode->privateIp), pDnode->customScore, score); + pDnode->customScore = score; + mgmtUpdateDnode(pDnode); + mgmtStartBalanceTimer(15); + } + return TSDB_CODE_INVALID_SQL; + } else if (strncasecmp(option, "bandwidth", 9) == 0) { + paGetToken(msg, &value, &valen); + if (valen > 0) { + int bandwidthMb = atoi(value); + if (bandwidthMb >= 0 && bandwidthMb < 10000000) { + mTrace("dnode:%s, bandwidth(Mb) set from:%d to:%d", taosIpStr(pDnode->privateIp), pDnode->bandwidthMb, bandwidthMb); + pDnode->bandwidthMb = bandwidthMb; + mgmtUpdateDnode(pDnode); + return TSDB_CODE_SUCCESS; + } + } + return TSDB_CODE_INVALID_SQL; + } + + return -1; +} + +int mgmtSendCfgDnodeMsg(char *cont) { + char * pMsg, *pStart; + int msgLen = 0; + SDnodeObj *pDnode; + SCfgMsg * pCfg = (SCfgMsg *)cont; + uint32_t ip; + + ip = inet_addr(pCfg->ip); + pDnode = mgmtGetDnode(ip); + if (pDnode == NULL) { + mError("dnode ip:%s not configured", pCfg->ip); + return TSDB_CODE_NOT_CONFIGURED; + } + + mTrace("dnode:%s, dynamic option received, content:%s", taosIpStr(pDnode->privateIp), pCfg->config); + int code = mgmtCfgDynamicOptions(pDnode, pCfg->config); + if (code != -1) { + return code; + } + + pStart = taosBuildReqMsg(pDnode->thandle, TSDB_MSG_TYPE_CFG_PNODE); + if (pStart == NULL) return TSDB_CODE_NODE_OFFLINE; + pMsg = pStart; + + memcpy(pMsg, cont, sizeof(SCfgMsg)); + pMsg += sizeof(SCfgMsg); + + msgLen = pMsg - pStart; + taosSendMsgToDnode(pDnode, pStart, msgLen); + + return 0; +} diff --git a/src/system/src/mgmtMeter.c b/src/system/detail/src/mgmtMeter.c old mode 100755 new mode 100644 similarity index 77% rename from src/system/src/mgmtMeter.c rename to src/system/detail/src/mgmtMeter.c index 9656ca66a6..bccf9a06c8 --- a/src/system/src/mgmtMeter.c +++ b/src/system/detail/src/mgmtMeter.c @@ -13,12 +13,14 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include #include #include #include "mgmt.h" +#include "mgmtUtil.h" #include "taosmsg.h" #include "tast.h" #include "textbuffer.h" @@ -31,11 +33,11 @@ extern int64_t sdbVersion; -#define mgmtDestroyMeter(pMeter) \ - do { \ - tfree(pMeter->schema); \ - pMeter->pSkipList = tSkipListDestroy((pMeter)->pSkipList);\ - tfree(pMeter); \ +#define mgmtDestroyMeter(pMeter) \ + do { \ + tfree(pMeter->schema); \ + pMeter->pSkipList = tSkipListDestroy((pMeter)->pSkipList); \ + tfree(pMeter); \ } while (0) enum _Meter_Update_Action { @@ -88,6 +90,11 @@ int32_t mgmtMeterModifyTagNameByName(STabObj *pMetric, const char *oname, const int32_t mgmtMeterModifyTagValueByName(STabObj *pMeter, char *tagName, char *nContent); int32_t mgmtMeterAddColumn(STabObj *pMeter, SSchema schema[], int ncols); int32_t mgmtMeterDropColumnByName(STabObj *pMeter, const char *name); +static int dropMeterImp(SDbObj *pDb, STabObj * pMeter, SAcctObj *pAcct); +static void dropAllMetersOfMetric(SDbObj *pDb, STabObj * pMetric, SAcctObj *pAcct); + +int mgmtCheckMeterLimit(SAcctObj *pAcct, SCreateTableMsg *pCreate); +int mgmtCheckMeterGrant(SCreateTableMsg *pCreate, STabObj * pMeter); void mgmtMeterActionInit() { mgmtMeterActionFp[SDB_TYPE_INSERT] = mgmtMeterActionInsert; @@ -102,24 +109,12 @@ void mgmtMeterActionInit() { mgmtMeterActionFp[SDB_TYPE_DESTROY] = mgmtMeterActionDestroy; } -static int32_t mgmtGetTagsLength(STabObj *pMetric, int32_t col) { // length befor column col - assert(mgmtIsMetric(pMetric) && col >= 0); - - int32_t len = 0; - for (int32_t i = 0; i < pMetric->numOfTags && i < col; ++i) { - len += ((SSchema *)pMetric->schema)[pMetric->numOfColumns + i].bytes; - } - - return len; -} - static int32_t mgmtGetReqTagsLength(STabObj *pMetric, int16_t *cols, int32_t numOfCols) { - assert(mgmtIsMetric(pMetric) && numOfCols >= 0 && numOfCols <= TSDB_MAX_TAGS); + assert(mgmtIsMetric(pMetric) && numOfCols >= 0 && numOfCols <= TSDB_MAX_TAGS + 1); int32_t len = 0; for (int32_t i = 0; i < numOfCols; ++i) { assert(cols[i] < pMetric->numOfTags); - if (cols[i] == -1) { len += TSDB_METER_NAME_LEN; } else { @@ -141,22 +136,6 @@ static void mgmtVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_ } } -static char *mgmtMeterGetTag(STabObj *pMeter, int32_t col, SSchema *pTagColSchema) { - if (!mgmtMeterCreateFromMetric(pMeter)) { - return NULL; - } - - STabObj *pMetric = mgmtGetMeter(pMeter->pTagData); - int32_t offset = mgmtGetTagsLength(pMetric, col) + TSDB_METER_ID_LEN; - assert(offset > 0); - - if (pTagColSchema != NULL) { - *pTagColSchema = ((SSchema *)pMetric->schema)[pMetric->numOfColumns + col]; - } - - return (pMeter->pTagData + offset); -} - void *mgmtMeterActionReset(void *row, char *str, int size, int *ssize) { STabObj *pMeter = (STabObj *)row; int tsize = pMeter->updateEnd - (char *)pMeter; @@ -199,7 +178,12 @@ void *mgmtMeterActionInsert(void *row, char *str, int size, int *ssize) { return NULL; } - pAcct = &acctObj; + pAcct = mgmtGetAcct(pDb->cfg.acct); + // TODO : check if account exists. + if (pAcct == NULL) { + mError("account not exists"); + return NULL; + } } if (mgmtMeterCreateFromMetric(pMeter)) { @@ -213,8 +197,14 @@ void *mgmtMeterActionInsert(void *row, char *str, int size, int *ssize) { } if (mgmtIsNormalMeter(pMeter)) { - if (pMetric) { - mgmtAddMeterIntoMetric(pMetric, pMeter); + if (pMetric) mgmtAddMeterIntoMetric(pMetric, pMeter); + + if (!sdbMaster) { + int sid = taosAllocateId(pVgroup->idPool); + if (sid != pMeter->gid.sid) { + mError("sid:%d is not matched from the master:%d", sid, pMeter->gid.sid); + return NULL; + } } pAcct->acctInfo.numOfTimeSeries += (pMeter->numOfColumns - 1); @@ -222,9 +212,7 @@ void *mgmtMeterActionInsert(void *row, char *str, int size, int *ssize) { pDb->numOfTables++; pVgroup->meterList[pMeter->gid.sid] = pMeter; - if (pVgroup->numOfMeters >= pDb->cfg.maxSessions - 1 && pDb->numOfVgroups > 1) { - mgmtMoveVgroupToTail(pDb, pVgroup); - } + if (pVgroup->numOfMeters >= pDb->cfg.maxSessions - 1 && pDb->numOfVgroups > 1) mgmtMoveVgroupToTail(pDb, pVgroup); } else { // insert a metric pMeter->pHead = NULL; @@ -347,7 +335,6 @@ void *mgmtMeterActionDecode(void *row, char *str, int size, int *ssize) { } memcpy(pMeter->schema, str + tsize, pMeter->schemaSize); - return (void *)pMeter; } @@ -491,13 +478,11 @@ int mgmtInitMeters() { if (mgmtMeterCreateFromMetric(pMeter)) { pMeter->pTagData = (char *)pMeter->schema; // + sizeof(SSchema)*pMeter->numOfColumns; pMetric = mgmtGetMeter(pMeter->pTagData); - if (pMetric) { - mgmtAddMeterIntoMetric(pMetric, pMeter); - } + if (pMetric) mgmtAddMeterIntoMetric(pMetric, pMeter); } - pAcct = &acctObj; - mgmtAddMeterStatisticToAcct(pMeter, pAcct); + pAcct = mgmtGetAcct(pDb->cfg.acct); + if (pAcct) mgmtAddMeterStatisticToAcct(pMeter, pAcct); } else { if (pDb) mgmtAddMetricIntoDb(pDb, pMeter); } @@ -516,6 +501,7 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { STabObj * pMetric = NULL; SVgObj * pVgroup = NULL; int size = 0; + SAcctObj *pAcct = NULL; int numOfTables = sdbGetNumOfRows(meterSdb); if (numOfTables >= tsMaxTables) { @@ -523,6 +509,13 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { return TSDB_CODE_TOO_MANY_TABLES; } + pAcct = mgmtGetAcct(pDb->cfg.acct); + assert(pAcct != NULL); + int code = mgmtCheckMeterLimit(pAcct, pCreate); + if (code != 0) { + return code; + } + // does table exist? pMeter = mgmtGetMeter(pCreate->meterId); if (pMeter) { @@ -547,10 +540,9 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { } /* - * for meters created according to metrics, the schema of this meter isn't - * needed. + * for meters created according to metrics, the schema of this meter isn't needed. * so, we don't allocate memory for it in order to save a huge amount of - * memory when a large amount of meters are created using metrics. + * memory when a large amount of meters are created according to this super table. */ size = mgmtGetTagsLength(pMetric, INT_MAX) + (uint32_t)TSDB_METER_ID_LEN; pMeter->schema = (char *)malloc(size); @@ -564,7 +556,7 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { pMeter->numOfColumns = pMetric->numOfColumns; pMeter->sversion = pMetric->sversion; - pMeter->pTagData = pMeter->schema; // + pMetric->numOfColumns*sizeof(SSchema); + pMeter->pTagData = pMeter->schema; pMeter->nextColId = pMetric->nextColId; memcpy(pMeter->pTagData, pTagData, size); @@ -612,6 +604,11 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { return TSDB_CODE_OTHERS; } + code = mgmtCheckMeterGrant(pCreate, pMeter); + if (code != 0) { + return code; + } + if (pCreate->numOfTags == 0) { // handle normal meter creation pVgroup = pDb->pHead; @@ -622,7 +619,7 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { if (pDb->vgStatus == TSDB_VG_STATUS_FULL) { mgmtDestroyMeter(pMeter); - return TSDB_CODE_NO_ENOUGH_PNODES; + return TSDB_CODE_NO_ENOUGH_DNODES; } if (pDb->vgStatus == TSDB_VG_STATUS_COMMITLOG_INIT_FAILED) { @@ -662,15 +659,15 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { pMeter->uid = (((uint64_t)pMeter->createdTime) << 16) + ((uint64_t)sdbVersion & ((1ul << 16) - 1ul)); } - if (sdbInsertRow(meterSdb, pMeter, 0) < 0) { - return TSDB_CODE_SDB_ERROR; - } + if (sdbInsertRow(meterSdb, pMeter, 0) < 0) return TSDB_CODE_SDB_ERROR; // send create message to the selected vnode servers if (pCreate->numOfTags == 0) { mTrace("meter:%s, send msg to dnode, vgId:%d, sid:%d, vnode:%d, dbname:%s", pMeter->meterId, pMeter->gid.vgId, pMeter->gid.sid, pVgroup->vnodeGid[0].vnode, pDb->name); - mgmtSendCreateMsgToVnode(pMeter, pVgroup->vnodeGid[0].vnode); + + grantAddTimeSeries(pMeter->numOfColumns - 1); + mgmtSendCreateMsgToVgroup(pMeter, pVgroup); } return 0; @@ -678,7 +675,6 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { int mgmtDropMeter(SDbObj *pDb, char *meterId, int ignore) { STabObj * pMeter; - SVgObj * pVgroup; SAcctObj *pAcct; pMeter = mgmtGetMeter(meterId); @@ -690,28 +686,25 @@ int mgmtDropMeter(SDbObj *pDb, char *meterId, int ignore) { } } - pAcct = &acctObj; + pAcct = mgmtGetAcct(pDb->cfg.acct); // 0.sys if (taosCheckDbName(pDb->name, tsMonitorDbName)) return TSDB_CODE_MONITOR_DB_FORBEIDDEN; if (mgmtIsNormalMeter(pMeter)) { - if (pAcct != NULL) pAcct->acctInfo.numOfTimeSeries -= (pMeter->numOfColumns - 1); - pVgroup = mgmtGetVgroup(pMeter->gid.vgId); - if (pVgroup == NULL) { - return TSDB_CODE_OTHERS; - } - - mgmtSendRemoveMeterMsgToVnode(pMeter, pVgroup->vnodeGid[0].vnode); - sdbDeleteRow(meterSdb, pMeter); - - if (pVgroup->numOfMeters <= 0) mgmtDropVgroup(pDb, pVgroup); + return dropMeterImp(pDb, pMeter, pAcct); } else { // remove a metric + /* if (pMeter->numOfMeters > 0) { assert(pMeter->pSkipList != NULL && pMeter->pSkipList->nSize > 0); return TSDB_CODE_RELATED_TABLES_EXIST; } + */ + // first delet all meters of metric + dropAllMetersOfMetric(pDb, pMeter, pAcct); + + // finally delete metric sdbDeleteRow(meterSdb, pMeter); } @@ -727,9 +720,7 @@ int mgmtAlterMeter(SDbObj *pDb, SAlterTableMsg *pAlter) { } // 0.sys - if (taosCheckDbName(pDb->name, tsMonitorDbName)) { - return TSDB_CODE_MONITOR_DB_FORBEIDDEN; - } + if (taosCheckDbName(pDb->name, tsMonitorDbName)) return TSDB_CODE_MONITOR_DB_FORBEIDDEN; if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { if (!mgmtIsNormalMeter(pMeter) || !mgmtMeterCreateFromMetric(pMeter)) { @@ -766,6 +757,31 @@ int mgmtAlterMeter(SDbObj *pDb, SAlterTableMsg *pAlter) { return TSDB_CODE_SUCCESS; } +static int dropMeterImp(SDbObj *pDb, STabObj * pMeter, SAcctObj *pAcct) { + SVgObj * pVgroup; + + if (pAcct != NULL) pAcct->acctInfo.numOfTimeSeries -= (pMeter->numOfColumns - 1); + + pVgroup = mgmtGetVgroup(pMeter->gid.vgId); + if (pVgroup == NULL) return TSDB_CODE_OTHERS; + + grantRestoreTimeSeries(pMeter->numOfColumns - 1); + mgmtSendRemoveMeterMsgToDnode(pMeter, pVgroup); + sdbDeleteRow(meterSdb, pMeter); + + if (pVgroup->numOfMeters <= 0) mgmtDropVgroup(pDb, pVgroup); + + return 0; +} + +static void dropAllMetersOfMetric(SDbObj *pDb, STabObj * pMetric, SAcctObj *pAcct) { + STabObj * pMeter = NULL; + + while ((pMeter = pMetric->pHead) != NULL) { + (void)dropMeterImp(pDb, pMeter, pAcct); + } +} + /* * create key of each meter for skip list, which is generated from first tag * column @@ -793,7 +809,6 @@ static void addMeterIntoMetricIndex(STabObj *pMetric, STabObj *pMeter) { if (pMetric->pSkipList) { tSkipListKey key = {0}; createKeyFromTagValue(pMetric, pMeter, &key); - tSkipListPut(pMetric->pSkipList, pMeter, &key, 1); tSkipListDestroyKey(&key); @@ -878,7 +893,7 @@ int mgmtGetMeterMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created time"); + strcpy(pSchema[cols].name, "created_time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -907,29 +922,6 @@ int mgmtGetMeterMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { return 0; } -static int32_t tabObjVGIDComparator(const void *pLeft, const void *pRight) { - STabObj *p1 = *(STabObj **)pLeft; - STabObj *p2 = *(STabObj **)pRight; - - int32_t ret = p1->gid.vgId - p2->gid.vgId; - if (ret == 0) { - return ret; - } else { - return ret > 0 ? 1 : -1; - } -} - -/* - * qsort comparator - * sort the result to ensure meters with the same gid is grouped together - */ -static int32_t nodeVGIDComparator(const void *pLeft, const void *pRight) { - tSkipListNode *p1 = *((tSkipListNode **)pLeft); - tSkipListNode *p2 = *((tSkipListNode **)pRight); - - return tabObjVGIDComparator(&p1->pData, &p2->pData); -} - SSchema *mgmtGetMeterSchema(STabObj *pMeter) { if (pMeter == NULL) { return NULL; @@ -964,7 +956,10 @@ static char *mgmtBuildMetricMetaMsg(STabObj *pMeter, int32_t *ovgId, SVnodeSidLi pMeta->numOfVnodes++; SVgObj *pVgroup = mgmtGetVgroup(pMeter->gid.vgId); - (*pList)->vpeerDesc[0].vnode = pVgroup->vnodeGid[0].vnode; + for (int i = 0; i < TSDB_VNODES_SUPPORT; ++i) { + (*pList)->vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp; + (*pList)->vpeerDesc[i].vnode = pVgroup->vnodeGid[i].vnode; + } pMsg += sizeof(SVnodeSidList); (*ovgId) = pMeter->gid.vgId; @@ -999,25 +994,13 @@ static char *mgmtBuildMetricMetaMsg(STabObj *pMeter, int32_t *ovgId, SVnodeSidLi return pMsg; } -static STabObj *mgmtGetResultPayload(tQueryResultset *pRes, int32_t index) { - if (index < 0 || index >= pRes->num) { - return NULL; - } - - if (pRes->nodeType == TAST_NODE_TYPE_INDEX_ENTRY) { - return (STabObj *)((tSkipListNode *)pRes->pRes[index])->pData; - } else { - return (STabObj *)pRes->pRes[index]; - } -} - // get total number of vnodes in final result set static int32_t mgmtGetNumOfVnodesInResult(tQueryResultset *pResult) { int32_t numOfVnodes = 0; int32_t prevGid = -1; for (int32_t i = 0; i < pResult->num; ++i) { - STabObj *pMeter = mgmtGetResultPayload(pResult, i); + STabObj *pMeter = pResult->pRes[i]; if (prevGid == -1) { prevGid = pMeter->gid.vgId; numOfVnodes++; @@ -1030,347 +1013,189 @@ static int32_t mgmtGetNumOfVnodesInResult(tQueryResultset *pResult) { return numOfVnodes; } -static void mgmtRetrieveMetersFromIDs(tQueryResultset *pRes, char *queryStr, char *origins, STabObj *pMetric) { - char * sep = ","; - char * pToken = NULL; - int32_t s = 4; - - pRes->pRes = malloc(sizeof(char *) * s); - pRes->nodeType = TAST_NODE_TYPE_METER_PTR; - pRes->num = 0; - - for (pToken = strsep(&queryStr, sep); pToken != NULL; pToken = strsep(&queryStr, sep)) { - STabObj *pMeterObj = mgmtGetMeter(pToken); - if (pMeterObj == NULL) { - mWarn("metric:%s error in metric query expression:%s, invalid meter id:%s", pMetric->meterId, origins, pToken); - continue; - } else { - /* double the old size */ - if (pRes->num >= s) { - s *= 2; - pRes->pRes = realloc(pRes->pRes, sizeof(char *) * s); - } - - /* not a table created from metric, ignore */ - if (pMeterObj->meterType != TSDB_METER_MTABLE) { - continue; - } +static int32_t mgmtGetMetricMetaMsgSize(tQueryResultset *pResult, int32_t tagLength, int32_t maxMetersPerQuery) { + int32_t numOfVnodes = mgmtGetNumOfVnodesInResult(pResult); - /* queried meter not belongs to this metric, ignore */ - if (mgmtGetMeter(pMeterObj->pTagData)->uid != pMetric->uid || - strncmp(pMetric->meterId, pMeterObj->pTagData, TSDB_METER_ID_LEN) != 0) { - continue; - } + int32_t size = (sizeof(SMeterSidExtInfo) + tagLength) * pResult->num + + ((pResult->num / maxMetersPerQuery) + 1 + numOfVnodes) * sizeof(SVnodeSidList) + sizeof(SMetricMeta) + + 1024; - pRes->pRes[pRes->num++] = pMeterObj; - } - } + return size; } -static int32_t tabObjResultComparator(const void *p1, const void *p2, void *param) { - tOrderDescriptor *pOrderDesc = (tOrderDescriptor *)param; +static SMetricMetaElemMsg *doConvertMetricMetaMsg(SMetricMetaMsg *pMetricMetaMsg, int32_t tableIndex) { + SMetricMetaElemMsg *pElem = (SMetricMetaElemMsg *)((char *)pMetricMetaMsg + pMetricMetaMsg->metaElem[tableIndex]); - STabObj *pNode1 = (STabObj *)p1; - STabObj *pNode2 = (STabObj *)p2; + pElem->orderIndex = htons(pElem->orderIndex); + pElem->orderType = htons(pElem->orderType); + pElem->numOfTags = htons(pElem->numOfTags); - for (int32_t i = 0; i < pOrderDesc->orderIdx.numOfOrderedCols; ++i) { - int32_t colIdx = pOrderDesc->orderIdx.pData[i]; + pElem->numOfGroupCols = htons(pElem->numOfGroupCols); + pElem->condLen = htonl(pElem->condLen); + pElem->cond = htonl(pElem->cond); - char *f1 = NULL; - char *f2 = NULL; + pElem->elemLen = htons(pElem->elemLen); - SSchema schema = {0}; + pElem->tableCond = htonl(pElem->tableCond); + pElem->tableCondLen = htonl(pElem->tableCondLen); - if (colIdx == -1) { - f1 = pNode1->meterId; - f2 = pNode2->meterId; - schema.type = TSDB_DATA_TYPE_BINARY; - schema.bytes = TSDB_METER_ID_LEN; - } else { - f1 = mgmtMeterGetTag(pNode1, colIdx, NULL); - f2 = mgmtMeterGetTag(pNode2, colIdx, &schema); - assert(schema.type == pOrderDesc->pTagSchema->pSchema[colIdx].type); - } - - int32_t ret = doCompare(f1, f2, schema.type, schema.bytes); - if (ret == 0) { - continue; - } else { - return ret; - } - } - - return 0; -} - -static int32_t nodeResultComparator(const void *p1, const void *p2, void *param) { - STabObj *pNode1 = (STabObj *)((tSkipListNode *)p1)->pData; - STabObj *pNode2 = (STabObj *)((tSkipListNode *)p2)->pData; - - return tabObjResultComparator(pNode1, pNode2, param); -} + pElem->rel = htons(pElem->rel); -// todo merge sort function with losertree used -static void mgmtReorganizeMetersInMetricMeta(STabObj *pMetric, SMetricMetaMsg *pInfo, SSchema *pTagSchema, - tQueryResultset *pRes) { - /* no result, no need to pagination */ - if (pRes->num <= 0) { - return; + for (int32_t i = 0; i < pElem->numOfTags; ++i) { + pElem->tagCols[i] = htons(pElem->tagCols[i]); } - /* - * To apply the group limitation and group offset, we should sort the result - * list according to the - * order condition - */ - tOrderDescriptor *descriptor = - (tOrderDescriptor *)calloc(1, sizeof(tOrderDescriptor) + sizeof(int32_t) * pInfo->numOfGroupbyCols); - descriptor->pTagSchema = tCreateTagSchema(pTagSchema, pMetric->numOfTags); - descriptor->orderIdx.numOfOrderedCols = pInfo->numOfGroupbyCols; - - int32_t *startPos = NULL; - int32_t numOfSubset = 1; - - if (pInfo->numOfGroupbyCols > 0) { - memcpy(descriptor->orderIdx.pData, (int16_t *)pInfo->groupbyTagIds, sizeof(int16_t) * pInfo->numOfGroupbyCols); - // sort results list - __ext_compar_fn_t comparFn = - (pRes->nodeType == TAST_NODE_TYPE_METER_PTR) ? tabObjResultComparator : nodeResultComparator; - - tQSortEx(pRes->pRes, POINTER_BYTES, 0, pRes->num - 1, descriptor, comparFn); - startPos = calculateSubGroup(pRes->pRes, pRes->num, &numOfSubset, descriptor, comparFn); - } else { - startPos = malloc(2 * sizeof(int32_t)); + pElem->groupbyTagColumnList = htonl(pElem->groupbyTagColumnList); - startPos[0] = 0; - startPos[1] = (int32_t)pRes->num; + int16_t *groupColIds = (int16_t*) (((char *)pMetricMetaMsg) + pElem->groupbyTagColumnList); + for (int32_t i = 0; i < pElem->numOfGroupCols; ++i) { + groupColIds[i] = htons(groupColIds[i]); } - /* - * sort the result according to vgid to ensure meters with the same vgid is - * continuous in the result list - */ - __compar_fn_t functor = (pRes->nodeType == TAST_NODE_TYPE_METER_PTR) ? tabObjVGIDComparator : nodeVGIDComparator; - qsort(pRes->pRes, (size_t) pRes->num, POINTER_BYTES, functor); - - free(descriptor->pTagSchema); - free(descriptor); - free(startPos); + return pElem; } -static char *getTagValueFromMeter(STabObj *pMeter, int32_t offset, void *param) { - if (offset == -1) { - extractMeterName(pMeter->meterId, param); - return param; - } else { - char *tags = pMeter->pTagData + TSDB_METER_ID_LEN; // tag start position - return (tags + offset); +static int32_t mgmtBuildMetricMetaRspMsg(void *thandle, SMetricMetaMsg *pMetricMetaMsg, tQueryResultset *pResult, + char **pStart, int32_t *tagLen, int32_t rspMsgSize, int32_t maxTablePerVnode, + int32_t code) { + *pStart = taosBuildRspMsgWithSize(thandle, TSDB_MSG_TYPE_METRIC_META_RSP, rspMsgSize); + if (*pStart == NULL) { + return 0; } -} -// todo refactor -bool tSQLElemFilterCallback(tSkipListNode *pNode, void *param) { - tQueryInfo *pCols = (tQueryInfo *)param; - STabObj * pMeter = (STabObj *)pNode->pData; + char * pMsg = (*pStart); + STaosRsp *pRsp = (STaosRsp *)pMsg; - char name[TSDB_METER_NAME_LEN + 1] = {0}; - char * val = getTagValueFromMeter(pMeter, pCols->offset, name); - int8_t type = (pCols->pSchema[pCols->colIdx].type); + pRsp->code = code; + pMsg += sizeof(STaosRsp); + *pMsg = TSDB_IE_TYPE_META; + pMsg++; - int32_t ret = 0; - if (pCols->q.nType == TSDB_DATA_TYPE_BINARY || pCols->q.nType == TSDB_DATA_TYPE_NCHAR) { - ret = pCols->comparator(val, pCols->q.pz); - } else { - tVariant v = {0}; - switch (type) { - case TSDB_DATA_TYPE_INT: - v.i64Key = *(int32_t *)val; - break; - case TSDB_DATA_TYPE_BIGINT: - v.i64Key = *(int64_t *)val; - break; - case TSDB_DATA_TYPE_TINYINT: - v.i64Key = *(int8_t *)val; - break; - case TSDB_DATA_TYPE_SMALLINT: - v.i64Key = *(int16_t *)val; - break; - case TSDB_DATA_TYPE_DOUBLE: - v.dKey = *(double *)val; - break; - case TSDB_DATA_TYPE_FLOAT: - v.dKey = *(float *)val; - break; - case TSDB_DATA_TYPE_BOOL: - v.i64Key = *(int8_t *)val; - break; - } - ret = pCols->comparator(&v.i64Key, &pCols->q.i64Key); + if (code != TSDB_CODE_SUCCESS) { + return pMsg - (*pStart); // one bit in payload } - switch (pCols->optr) { - case TSDB_RELATION_EQUAL: { - return ret == 0; - } - case TSDB_RELATION_NOT_EQUAL: { - return ret != 0; - } - case TSDB_RELATION_LARGE_EQUAL: { - return ret >= 0; - } - case TSDB_RELATION_LARGE: { - return ret > 0; - } - case TSDB_RELATION_LESS_EQUAL: { - return ret <= 0; - } - case TSDB_RELATION_LESS: { - return ret < 0; - } - case TSDB_RELATION_LIKE: { - return ret == 0; - } - - default: - assert(false); - } - return true; -} + int32_t msgLen = 0; -int mgmtRetrieveMetersFromMetric(STabObj *pMetric, SMetricMetaMsg *pInfo, tQueryResultset *pRes) { - /* no table created in accordance with this metric. */ - if (pMetric->pSkipList == NULL || pMetric->pSkipList->nSize == 0) { - assert(pMetric->numOfMeters == 0); - return TSDB_CODE_SUCCESS; - } + *(int16_t *)pMsg = htons(pMetricMetaMsg->numOfMeters); + pMsg += sizeof(int16_t); - char * pQueryCond = pInfo->tags; - int32_t queryCondLength = pInfo->condLength; + for (int32_t j = 0; j < pMetricMetaMsg->numOfMeters; ++j) { + SVnodeSidList *pList = NULL; + int ovgId = -1; - tSQLBinaryExpr *pExpr = NULL; - SSchema * pTagSchema = (SSchema *)(pMetric->schema + pMetric->numOfColumns * sizeof(SSchema)); + SMetricMeta *pMeta = (SMetricMeta *)pMsg; - char *queryStr = calloc(1, (queryCondLength + 1) * TSDB_NCHAR_SIZE); - if (queryCondLength > 0) { - /* transfer the unicode string to mbs binary expression */ - taosUcs4ToMbs(pQueryCond, queryCondLength * TSDB_NCHAR_SIZE, queryStr); - queryCondLength = strlen(queryStr) + 1; + pMeta->numOfMeters = 0; + pMeta->numOfVnodes = 0; + pMeta->tagLen = htons((uint16_t)tagLen[j]); - mTrace("metric:%s len:%d, type:%d, metric query condition:%s", pMetric->meterId, queryCondLength, pInfo->type, queryStr); - } else { - mTrace("metric:%s, retrieve all meter, no query condition", pMetric->meterId); - } + pMsg = (char *)pMeta + sizeof(SMetricMeta); - if (queryCondLength > 0) { - if (pInfo->type == TSQL_STABLE_QTYPE_SET) { - char *oldStr = strdup(queryStr); - mgmtRetrieveMetersFromIDs(pRes, queryStr, oldStr, pMetric); - tfree(oldStr); - } else { - tSQLBinaryExprFromString(&pExpr, pTagSchema, pMetric->numOfTags, queryStr, queryCondLength); + SMetricMetaElemMsg *pElem = (SMetricMetaElemMsg *)((char *)pMetricMetaMsg + pMetricMetaMsg->metaElem[j]); - /* failed to build expression, no result, return immediately */ - if (pExpr == NULL) { - mError("metric:%s, no result returned, error in metric query expression:%s", pMetric->meterId, queryStr); - tfree(queryStr); - return TSDB_CODE_OPS_NOT_SUPPORT; - } else { - // query according to the binary expression - tSQLBinaryExprTraverse(pExpr, pMetric->pSkipList, pTagSchema, pMetric->numOfTags, tSQLElemFilterCallback, pRes); - tSQLBinaryExprDestroy(&pExpr); - } + for (int32_t i = 0; i < pResult[j].num; ++i) { + STabObj *pMeter = pResult[j].pRes[i]; + pMsg = mgmtBuildMetricMetaMsg(pMeter, &ovgId, &pList, pMeta, tagLen[j], pElem->numOfTags, pElem->tagCols, + maxTablePerVnode, pMsg); } - } else { - pRes->num = tSkipListIterateList(pMetric->pSkipList, (tSkipListNode ***)&pRes->pRes, NULL, NULL); - } - tfree(queryStr); - mTrace("metric:%s numOfRes:%d", pMetric->meterId, pRes->num); + mTrace("metric:%s metric-meta tables:%d, vnode:%d", pElem->meterId, pMeta->numOfMeters, pMeta->numOfVnodes); - mgmtReorganizeMetersInMetricMeta(pMetric, pInfo, pTagSchema, pRes); - return TSDB_CODE_SUCCESS; -} - -static int32_t mgmtGetMetricMetaMsgSize(tQueryResultset *pResult, int32_t tagLength, int32_t maxMetersPerQuery) { - int32_t numOfVnodes = mgmtGetNumOfVnodesInResult(pResult); + pMeta->numOfMeters = htonl(pMeta->numOfMeters); + pMeta->numOfVnodes = htonl(pMeta->numOfVnodes); + } - int32_t size = (sizeof(SMeterSidExtInfo) + tagLength) * pResult->num + - ((pResult->num / maxMetersPerQuery) + 1 + numOfVnodes) * sizeof(SVnodeSidList) + sizeof(SMetricMeta) + - 1024; + msgLen = pMsg - (*pStart); + mTrace("metric-meta msg size %d", msgLen); - return size; + return msgLen; } -int mgmtRetrieveMetricMeta(void *thandle, char **pStart, STabObj *pMetric, SMetricMetaMsg *pMetricMetaMsg) { - SVnodeSidList *pList = NULL; - - int32_t tagLen = mgmtGetReqTagsLength(pMetric, (int16_t *)pMetricMetaMsg->tagCols, pMetricMetaMsg->numOfTags); - +int mgmtRetrieveMetricMeta(void *thandle, char **pStart, SMetricMetaMsg *pMetricMetaMsg) { /* * naive method: Do not limit the maximum number of meters in each - * vnode(subquery), - * split the result according to vnodes + * vnode(subquery), split the result according to vnodes + * * todo: split the number of vnodes to make sure each vnode has the same - * number of - * tables to query, while not break the upper limit of number of vnode queries + * number of tables to query, while not break the upper limit of number of vnode queries */ - int32_t maxMetersPerVNodeInQuery = INT32_MAX; + int32_t maxMetersPerVNodeForQuery = INT32_MAX; + int msgLen = 0; + int ret = TSDB_CODE_SUCCESS; + tQueryResultset *result = calloc(1, pMetricMetaMsg->numOfMeters * sizeof(tQueryResultset)); + int32_t * tagLen = calloc(1, sizeof(int32_t) * pMetricMetaMsg->numOfMeters); + + if (result == NULL || tagLen == NULL) { + return -1; + } - int ovgId = -1; + for (int32_t i = 0; i < pMetricMetaMsg->numOfMeters; ++i) { + SMetricMetaElemMsg *pElem = doConvertMetricMetaMsg(pMetricMetaMsg, i); + STabObj * pMetric = mgmtGetMeter(pElem->meterId); - tQueryResultset result = {0}; - int ret = mgmtRetrieveMetersFromMetric(pMetric, pMetricMetaMsg, &result); + if (!mgmtIsMetric(pMetric)) { + ret = TSDB_CODE_NOT_SUPER_TABLE; + break; + } - int rspMsgSize = 512; - if (ret == TSDB_CODE_SUCCESS) { - rspMsgSize = mgmtGetMetricMetaMsgSize(&result, tagLen, maxMetersPerVNodeInQuery); + tagLen[i] = mgmtGetReqTagsLength(pMetric, (int16_t *)pElem->tagCols, pElem->numOfTags); } - *pStart = taosBuildRspMsgWithSize(thandle, TSDB_MSG_TYPE_METRIC_META_RSP, rspMsgSize); - if (*pStart == NULL) return 0; +#if 0 + //todo: opt for join process + int64_t num = 0; + int32_t index = 0; - char * pMsg = (*pStart); - STaosRsp *pRsp = (STaosRsp *)pMsg; + for (int32_t i = 0; i < pMetricMetaMsg->numOfMeters; ++i) { + SMetricMetaElemMsg *pElem = (SMetricMetaElemMsg*) ((char *) pMetricMetaMsg + pMetricMetaMsg->metaElem[i]); + STabObj *pMetric = mgmtGetMeter(pElem->meterId); - pRsp->code = ret; - pMsg += sizeof(STaosRsp); - *pMsg = TSDB_IE_TYPE_META; - pMsg++; + if (pMetric->pSkipList->nSize > num) { + index = i; + num = pMetric->pSkipList->nSize; + } + } +#endif - if (ret != TSDB_CODE_SUCCESS) { - return pMsg - (*pStart); // one bit in payload + if (ret == TSDB_CODE_SUCCESS) { + for (int32_t i = 0; i < pMetricMetaMsg->numOfMeters; ++i) { + ret = mgmtRetrieveMetersFromMetric(pMetricMetaMsg, i, &result[i]); + // todo opt performance + // if (result[i].num <= 0) {//no result + // } else if (result[i].num < 10) { + // } + } } - SMetricMeta *pMeta = (SMetricMeta *)pMsg; - - pMeta->numOfMeters = 0; - pMeta->numOfVnodes = 0; - pMeta->tagLen = htons((uint16_t)tagLen); - - pMsg = (char *)pMeta + sizeof(SMetricMeta); + if (ret == TSDB_CODE_SUCCESS) { + ret = mgmtDoJoin(pMetricMetaMsg, result); + } - // char* start = pMsg; - for (int32_t i = 0; i < result.num; ++i) { - STabObj *pMeter = mgmtGetResultPayload(&result, i); + if (ret == TSDB_CODE_SUCCESS) { + for (int32_t i = 0; i < pMetricMetaMsg->numOfMeters; ++i) { + mgmtReorganizeMetersInMetricMeta(pMetricMetaMsg, i, &result[i]); + } + } -#ifdef _DEBUG_VIEW - mTrace("vgid: %d, sid: %d", pMeter->gid.vgId, pMeter->gid.sid); -#endif - pMsg = mgmtBuildMetricMetaMsg(pMeter, &ovgId, &pList, pMeta, tagLen, pMetricMetaMsg->numOfTags, - pMetricMetaMsg->tagCols, maxMetersPerVNodeInQuery, pMsg); + if (ret == TSDB_CODE_SUCCESS) { + for (int32_t i = 0; i < pMetricMetaMsg->numOfMeters; ++i) { + msgLen += mgmtGetMetricMetaMsgSize(&result[i], tagLen[i], maxMetersPerVNodeForQuery); + } + } else { + msgLen = 512; } - // char* output = malloc(pMsg - (*pStart)); - // int32_t len = tsCompressString(start, pMsg - start, 0, output, 2, NULL); + msgLen = mgmtBuildMetricMetaRspMsg(thandle, pMetricMetaMsg, result, pStart, tagLen, msgLen, maxMetersPerVNodeForQuery, + ret); - int32_t msgLen = pMsg - (*pStart); - mTrace("metric:%s metric-meta tables:%d, vnode:%d, msg size %d", pMetric->meterId, pMeta->numOfMeters, - pMeta->numOfVnodes, msgLen); + for (int32_t i = 0; i < pMetricMetaMsg->numOfMeters; ++i) { + tQueryResultClean(&result[i]); + } - pMeta->numOfMeters = htonl(pMeta->numOfMeters); - pMeta->numOfVnodes = htonl(pMeta->numOfVnodes); + free(tagLen); + free(result); - tfree(result.pRes); return msgLen; } @@ -1456,7 +1281,7 @@ int mgmtGetMetricMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created time"); + strcpy(pSchema[cols].name, "created_time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -1787,7 +1612,12 @@ int32_t mgmtMeterAddColumn(STabObj *pMeter, SSchema schema[], int ncols) { return TSDB_CODE_APP_ERROR; } - pAcct = &acctObj; + pAcct = mgmtGetAcct(pDb->cfg.acct); + if (pAcct == NULL) { + mError("DB: %s not belongs to andy account", pDb->name); + return TSDB_CODE_APP_ERROR; + } + pMeter->schema = realloc(pMeter->schema, pMeter->schemaSize + sizeof(SSchema) * ncols); if (pMeter->meterType == TSDB_METER_OTABLE) { @@ -1838,7 +1668,11 @@ int32_t mgmtMeterDropColumnByName(STabObj *pMeter, const char *name) { return TSDB_CODE_APP_ERROR; } - pAcct = &acctObj; + pAcct = mgmtGetAcct(pDb->cfg.acct); + if (pAcct == NULL) { + mError("DB: %s not belongs to any account", pDb->name); + return TSDB_CODE_APP_ERROR; + } if (pMeter->meterType == TSDB_METER_OTABLE) { memmove(pMeter->schema + sizeof(SSchema) * index, pMeter->schema + sizeof(SSchema) * (index + 1), diff --git a/src/system/src/mgmtProfile.c b/src/system/detail/src/mgmtProfile.c similarity index 99% rename from src/system/src/mgmtProfile.c rename to src/system/detail/src/mgmtProfile.c index 6b612b75a0..e641739e31 100644 --- a/src/system/src/mgmtProfile.c +++ b/src/system/detail/src/mgmtProfile.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "mgmt.h" #include #include "mgmtProfile.h" @@ -44,7 +45,7 @@ typedef struct { int mgmtSaveQueryStreamList(char *cont, int contLen, SConnObj *pConn) { SAcctObj *pAcct = pConn->pAcct; - if (contLen <= 0) { + if (contLen <= 0 || pAcct == NULL) { return 0; } @@ -134,7 +135,7 @@ int mgmtGetQueryMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created time"); + strcpy(pSchema[cols].name, "created_time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; diff --git a/src/system/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c similarity index 66% rename from src/system/src/mgmtShell.c rename to src/system/detail/src/mgmtShell.c index 9621af763b..44b9ff286c 100644 --- a/src/system/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -13,23 +13,41 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include -#include "taosmsg.h" #include "dnodeSystem.h" #include "mgmt.h" #include "mgmtProfile.h" +#include "taosmsg.h" #include "tlog.h" -#pragma GCC diagnostic ignored "-Wint-conversion" +#pragma GCC diagnostic push + +#pragma GCC diagnostic ignored "-Woverflow" #pragma GCC diagnostic ignored "-Wpointer-sign" +#pragma GCC diagnostic ignored "-Wint-conversion" + +#define MAX_LEN_OF_METER_META (sizeof(SMultiMeterMeta) + sizeof(SSchema) * TSDB_MAX_COLUMNS + sizeof(SSchema) * TSDB_MAX_TAGS + TSDB_MAX_TAGS_LEN) void * pShellConn = NULL; SConnObj *connList; -void *mgmtProcessMsgFromShell(char *msg, void *ahandle, void *thandle); -int (*mgmtProcessShellMsg[TSDB_MSG_TYPE_MAX])(char *, int, SConnObj *); -void mgmtInitProcessShellMsg(); -int mgmtKillQuery(char *queryId, SConnObj *pConn); +void * mgmtProcessMsgFromShell(char *msg, void *ahandle, void *thandle); +int (*mgmtProcessShellMsg[TSDB_MSG_TYPE_MAX])(char *, int, SConnObj *); +void mgmtInitProcessShellMsg(); +int mgmtRedirectMsg(SConnObj *pConn, int msgType); +int mgmtKillQuery(char *queryId, SConnObj *pConn); + +int mgmtCheckRedirectMsg(SConnObj *pConn, int msgType); +int mgmtProcessAlterAcctMsg(char *pMsg, int msgLen, SConnObj *pConn); +int mgmtProcessCreateMnodeMsg(char *pMsg, int msgLen, SConnObj *pConn); +int mgmtProcessCreateDnodeMsg(char *pMsg, int msgLen, SConnObj *pConn); +int mgmtProcessCfgMnodeMsg(char *pMsg, int msgLen, SConnObj *pConn); +int mgmtProcessDropMnodeMsg(char *pMsg, int msgLen, SConnObj *pConn); +int mgmtProcessDropDnodeMsg(char *pMsg, int msgLen, SConnObj *pConn); +int mgmtProcessDropAcctMsg(char *pMsg, int msgLen, SConnObj *pConn); +int mgmtProcessCreateAcctMsg(char *pMsg, int msgLen, SConnObj *pConn); +int mgmtProcessCfgDnodeMsg(char *pMsg, int msgLen, SConnObj *pConn); void mgmtProcessTranRequest(SSchedMsg *pSchedMsg) { SIntMsg * pMsg = (SIntMsg *)(pSchedMsg->msg); @@ -37,7 +55,8 @@ void mgmtProcessTranRequest(SSchedMsg *pSchedMsg) { char *cont = (char *)pMsg->content + sizeof(SMgmtHead); int contLen = pMsg->msgLen - sizeof(SIntMsg) - sizeof(SMgmtHead); - (*mgmtProcessShellMsg[pMsg->msgType])(cont, contLen, pConn); + + if (pConn->pAcct) (*mgmtProcessShellMsg[pMsg->msgType])(cont, contLen, pConn); if (pSchedMsg->msg) free(pSchedMsg->msg); } @@ -49,13 +68,21 @@ int mgmtInitShell() { int size = sizeof(SConnObj) * tsMaxShellConns; connList = (SConnObj *)malloc(size); + if (connList == NULL) { + mError("failed to malloc for connList to shell"); + return -1; + } memset(connList, 0, size); int numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore / 4.0; if (numOfThreads < 1) numOfThreads = 1; memset(&rpcInit, 0, sizeof(rpcInit)); +#ifdef CLUSTER + rpcInit.localIp = tsInternalIp; +#else rpcInit.localIp = "0.0.0.0"; +#endif rpcInit.localPort = tsMgmtShellPort; rpcInit.label = "MND-shell"; rpcInit.numOfThreads = numOfThreads; @@ -118,6 +145,15 @@ static char *mgmtAllocMsg(SConnObj *pConn, int32_t size, char **pMsg, STaosRsp * return pStart; } +static char *mgmtForMultiAllocMsg(SConnObj *pConn, int32_t size, char **pMsg, STaosRsp **pRsp) { + char *pStart = taosBuildRspMsgWithSize(pConn->thandle, TSDB_MSG_TYPE_MULTI_METERINFO_RSP, size); + if (pStart == NULL) return 0; + *pMsg = pStart; + *pRsp = (STaosRsp *)(*pMsg); + + return pStart; +} + /** * check if we need to add mgmtProcessMeterMetaMsg into tranQueue, which will be executed one-by-one. * @@ -153,10 +189,12 @@ int mgmtProcessMeterMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { int size = sizeof(STaosHeader) + sizeof(STaosRsp) + sizeof(SMeterMeta) + sizeof(SSchema) * TSDB_MAX_COLUMNS + sizeof(SSchema) * TSDB_MAX_TAGS + TSDB_MAX_TAGS_LEN + TSDB_EXTRA_PAYLOAD_SIZE; + // todo db check should be extracted if (pConn->pDb == NULL || (pConn->pDb != NULL && pConn->pDb->dropStatus != TSDB_DB_STATUS_READY)) { - // todo handle failed to allocate msg buffer + if ((pStart = mgmtAllocMsg(pConn, size, &pMsg, &pRsp)) == NULL) { - return 0; + taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_METERINFO_RSP, TSDB_CODE_SERV_OUT_OF_MEMORY); + return 0; } pRsp->code = TSDB_CODE_INVALID_DB; @@ -169,8 +207,14 @@ int mgmtProcessMeterMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { // on demand create table from super table if meter does not exists if (pMeterObj == NULL && pInfo->createFlag == 1) { + // write operation needs to redirect to master mnode + if (mgmtCheckRedirectMsg(pConn, TSDB_MSG_TYPE_METERINFO_RSP) != 0) { + return 0; + } + SCreateTableMsg *pCreateMsg = calloc(1, sizeof(SCreateTableMsg) + sizeof(STagData)); if (pCreateMsg == NULL) { + taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_METERINFO_RSP, TSDB_CODE_SERV_OUT_OF_MEMORY); return 0; } @@ -193,7 +237,8 @@ int mgmtProcessMeterMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { if (code != TSDB_CODE_SUCCESS) { if ((pStart = mgmtAllocMsg(pConn, size, &pMsg, &pRsp)) == NULL) { - return 0; + taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_METERINFO_RSP, TSDB_CODE_SERV_OUT_OF_MEMORY); + return 0; } pRsp->code = code; @@ -206,6 +251,7 @@ int mgmtProcessMeterMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { } if ((pStart = mgmtAllocMsg(pConn, size, &pMsg, &pRsp)) == NULL) { + taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_METERINFO_RSP, TSDB_CODE_SERV_OUT_OF_MEMORY); return 0; } @@ -226,13 +272,13 @@ int mgmtProcessMeterMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { pMeta->uid = htobe64(pMeterObj->uid); pMeta->sid = htonl(pMeterObj->gid.sid); pMeta->vgid = htonl(pMeterObj->gid.vgId); - pMeta->sversion = htonl(pMeterObj->sversion); + pMeta->sversion = htons(pMeterObj->sversion); - pMeta->precision = htons(pConn->pDb->cfg.precision); + pMeta->precision = pConn->pDb->cfg.precision; - pMeta->numOfTags = htons(pMeterObj->numOfTags); + pMeta->numOfTags = pMeterObj->numOfTags; pMeta->numOfColumns = htons(pMeterObj->numOfColumns); - pMeta->meterType = htons(pMeterObj->meterType); + pMeta->meterType = pMeterObj->meterType; pMsg += sizeof(SMeterMeta); pSchema = (SSchema *)pMsg; // schema locates at the end of SMeterMeta struct @@ -243,13 +289,12 @@ int mgmtProcessMeterMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { STabObj *pMetric = mgmtGetMeter(pMeterObj->pTagData); uint32_t numOfTotalCols = (uint32_t)pMetric->numOfTags + pMetric->numOfColumns; - pMeta->numOfTags = htons(pMetric->numOfTags); // update the numOfTags info + pMeta->numOfTags = pMetric->numOfTags; // update the numOfTags info mgmtSetSchemaFromMeters(pSchema, pMetric, numOfTotalCols); pMsg += numOfTotalCols * sizeof(SSchema); // for meters created from metric, we need the metric tag schema to parse the tag data int32_t tagsLen = mgmtSetMeterTagValue(pMsg, pMetric, pMeterObj); - pMsg += tagsLen; } else { /* @@ -282,17 +327,190 @@ _exit_code: return msgLen; } +/** + * multi meter meta rsp pkg format: + * | STaosRsp | ieType | SMultiMeterInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 + * 1B 1B 4B + * + * | STaosHeader | STaosRsp | ieType | SMultiMeterInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | ......................| + * ^ ^ ^ + * |<--------------------------------------size-----------------------------------------------|---------------------->| + * | | | + * pStart pCurMeter pTail + **/ +int mgmtProcessMultiMeterMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { + SDbObj * pDbObj = NULL; + STabObj * pMeterObj = NULL; + SVgObj * pVgroup = NULL; + SMultiMeterMeta * pMeta = NULL; + SSchema * pSchema = NULL; + STaosRsp * pRsp = NULL; + char * pStart = NULL; + + SMultiMeterInfoMsg * pInfo = (SMultiMeterInfoMsg *)pMsg; + char * str = pMsg + sizeof(SMultiMeterInfoMsg); + pInfo->numOfMeters = htonl(pInfo->numOfMeters); + + int size = 4*1024*1024; // first malloc 4 MB, subsequent reallocation as twice + + char *pNewMsg; + if ((pStart = mgmtForMultiAllocMsg(pConn, size, &pNewMsg, &pRsp)) == NULL) { + taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_MULTI_METERINFO_RSP, TSDB_CODE_SERV_OUT_OF_MEMORY); + return 0; + } + + int32_t totalNum = 0; + char tblName[TSDB_METER_ID_LEN]; + char* nextStr; + + char* pCurMeter = pStart + sizeof(STaosRsp) + sizeof(SMultiMeterInfoMsg) + 1; // 1: ie type byte + char* pTail = pStart + size; + + while (str - pMsg < msgLen) { + nextStr = strchr(str, ','); + if (nextStr == NULL) { + break; + } + + memcpy(tblName, str, nextStr - str); + tblName[nextStr - str] = '\0'; + str = nextStr + 1; + + // judge whether the remaining memory is adequate + if ((pTail - pCurMeter) < MAX_LEN_OF_METER_META) { + char* pMsgHdr = pStart - sizeof(STaosHeader); + size *= 2; + pMsgHdr = (char*)realloc(pMsgHdr, size); + if (NULL == pMsgHdr) { + char* pTmp = pStart - sizeof(STaosHeader); + tfree(pTmp); + taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_MULTI_METERINFO_RSP, TSDB_CODE_SERV_OUT_OF_MEMORY); + break; + } + + pCurMeter = (char*)pMsgHdr + sizeof(STaosHeader) + (pCurMeter - pStart); + pStart = (char*)pMsgHdr + sizeof(STaosHeader); + pNewMsg = pStart; + pRsp = (STaosRsp *)pStart; + pTail = pMsgHdr + size; + } + + // get meter schema, and fill into resp payload + pMeterObj = mgmtGetMeter(tblName); + pDbObj = mgmtGetDbByMeterId(tblName); + + if (pMeterObj == NULL || (pDbObj == NULL)) { + continue; + } else { + mTrace("%s, uid:%lld sversion:%d meter meta is retrieved", tblName, pMeterObj->uid, pMeterObj->sversion); + pMeta = (SMultiMeterMeta *)pCurMeter; + + memcpy(pMeta->meterId, tblName, strlen(tblName)); + pMeta->meta.uid = htobe64(pMeterObj->uid); + pMeta->meta.sid = htonl(pMeterObj->gid.sid); + pMeta->meta.vgid = htonl(pMeterObj->gid.vgId); + pMeta->meta.sversion = htons(pMeterObj->sversion); + pMeta->meta.precision = pDbObj->cfg.precision; + pMeta->meta.numOfTags = pMeterObj->numOfTags; + pMeta->meta.numOfColumns = htons(pMeterObj->numOfColumns); + pMeta->meta.meterType = pMeterObj->meterType; + + pCurMeter += sizeof(SMultiMeterMeta); + pSchema = (SSchema *)pCurMeter; // schema locates at the end of SMeterMeta struct + + if (mgmtMeterCreateFromMetric(pMeterObj)) { + assert(pMeterObj->numOfTags == 0); + + STabObj *pMetric = mgmtGetMeter(pMeterObj->pTagData); + uint32_t numOfTotalCols = (uint32_t)pMetric->numOfTags + pMetric->numOfColumns; + + pMeta->meta.numOfTags = pMetric->numOfTags; // update the numOfTags info + mgmtSetSchemaFromMeters(pSchema, pMetric, numOfTotalCols); + pCurMeter += numOfTotalCols * sizeof(SSchema); + + // for meters created from metric, we need the metric tag schema to parse the tag data + int32_t tagsLen = mgmtSetMeterTagValue(pCurMeter, pMetric, pMeterObj); + pCurMeter += tagsLen; + } else { + /* + * for metrics, or meters that are not created from metric, set the schema directly + * for meters created from metric, we use the schema of metric instead + */ + uint32_t numOfTotalCols = (uint32_t)pMeterObj->numOfTags + pMeterObj->numOfColumns; + mgmtSetSchemaFromMeters(pSchema, pMeterObj, numOfTotalCols); + pCurMeter += numOfTotalCols * sizeof(SSchema); + } + + if (mgmtIsNormalMeter(pMeterObj)) { + pVgroup = mgmtGetVgroup(pMeterObj->gid.vgId); + if (pVgroup == NULL) { + pRsp->code = TSDB_CODE_INVALID_TABLE; + pNewMsg++; + mError("%s, uid:%lld sversion:%d vgId:%d pVgroup is NULL", tblName, pMeterObj->uid, pMeterObj->sversion, + pMeterObj->gid.vgId); + goto _error_exit_code; + } + + for (int i = 0; i < TSDB_VNODES_SUPPORT; ++i) { + pMeta->meta.vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp; + pMeta->meta.vpeerDesc[i].vnode = htonl(pVgroup->vnodeGid[i].vnode); + } + } + } + + totalNum++; + if (totalNum > pInfo->numOfMeters) { + pNewMsg++; + break; + } + } + + // fill rsp code, ieType + msgLen = pCurMeter - pNewMsg; + + pRsp->code = 0; + pNewMsg += sizeof(STaosRsp); + *pNewMsg = TSDB_IE_TYPE_META; + pNewMsg++; + + SMultiMeterInfoMsg *pRspInfo = (SMultiMeterInfoMsg *)pNewMsg; + + pRspInfo->numOfMeters = htonl(totalNum); + goto _exit_code; + +_error_exit_code: + msgLen = pNewMsg - pStart; + +_exit_code: + taosSendMsgToPeer(pConn->thandle, pStart, msgLen); + + return msgLen; +} + int mgmtProcessMetricMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { SMetricMetaMsg *pMetricMetaMsg = (SMetricMetaMsg *)pMsg; STabObj * pMetric; STaosRsp * pRsp; char * pStart; - pMetric = mgmtGetMeter(pMetricMetaMsg->meterId); + pMetricMetaMsg->numOfMeters = htonl(pMetricMetaMsg->numOfMeters); + + pMetricMetaMsg->join = htonl(pMetricMetaMsg->join); + pMetricMetaMsg->joinCondLen = htonl(pMetricMetaMsg->joinCondLen); + + for (int32_t i = 0; i < pMetricMetaMsg->numOfMeters; ++i) { + pMetricMetaMsg->metaElem[i] = htonl(pMetricMetaMsg->metaElem[i]); + } + + SMetricMetaElemMsg *pElem = (SMetricMetaElemMsg *)(((char *)pMetricMetaMsg) + pMetricMetaMsg->metaElem[0]); + pMetric = mgmtGetMeter(pElem->meterId); if (pMetric == NULL || (pConn->pDb != NULL && pConn->pDb->dropStatus != TSDB_DB_STATUS_READY)) { pStart = taosBuildRspMsg(pConn->thandle, TSDB_MSG_TYPE_METRIC_META_RSP); - if (pStart == NULL) return 0; + if (pStart == NULL) { + taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_METRIC_META_RSP, TSDB_CODE_SERV_OUT_OF_MEMORY); + return 0; + } pMsg = pStart; pRsp = (STaosRsp *)pMsg; @@ -304,28 +522,11 @@ int mgmtProcessMetricMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { msgLen = pMsg - pStart; } else { - pMetricMetaMsg->condLength = htonl(pMetricMetaMsg->condLength); - pMetricMetaMsg->orderIndex = htons(pMetricMetaMsg->orderIndex); - pMetricMetaMsg->orderType = htons(pMetricMetaMsg->orderType); - pMetricMetaMsg->numOfTags = htons(pMetricMetaMsg->numOfTags); - - pMetricMetaMsg->type = htons(pMetricMetaMsg->type); - pMetricMetaMsg->numOfGroupbyCols = htons(pMetricMetaMsg->numOfGroupbyCols); - - pMetricMetaMsg->groupbyTagIds = ((char *)pMetricMetaMsg->tags) + pMetricMetaMsg->condLength * TSDB_NCHAR_SIZE; - int16_t *groupbyColIds = (int16_t *)pMetricMetaMsg->groupbyTagIds; - for (int32_t i = 0; i < pMetricMetaMsg->numOfGroupbyCols; ++i) { - groupbyColIds[i] = htons(groupbyColIds[i]); - } - - for (int32_t i = 0; i < pMetricMetaMsg->numOfTags; ++i) { - pMetricMetaMsg->tagCols[i] = htons(pMetricMetaMsg->tagCols[i]); + msgLen = mgmtRetrieveMetricMeta(pConn->thandle, &pStart, pMetricMetaMsg); + if (msgLen <= 0) { + taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_METRIC_META_RSP, TSDB_CODE_SERV_OUT_OF_MEMORY); + return 0; } - - pMetricMetaMsg->limit = htobe64(pMetricMetaMsg->limit); - pMetricMetaMsg->offset = htobe64(pMetricMetaMsg->offset); - - msgLen = mgmtRetrieveMetricMeta(pConn->thandle, &pStart, pMetric, pMetricMetaMsg); } taosSendMsgToPeer(pConn->thandle, pStart, msgLen); @@ -337,6 +538,10 @@ int mgmtProcessCreateDbMsg(char *pMsg, int msgLen, SConnObj *pConn) { SCreateDbMsg *pCreate = (SCreateDbMsg *)pMsg; int code = 0; + if (mgmtCheckRedirectMsg(pConn, TSDB_MSG_TYPE_CREATE_DB_RSP) != 0) { + return 0; + } + pCreate->maxSessions = htonl(pCreate->maxSessions); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); // pCreate->cacheNumOfBlocks = htonl(pCreate->cacheNumOfBlocks); @@ -348,11 +553,12 @@ int mgmtProcessCreateDbMsg(char *pMsg, int msgLen, SConnObj *pConn) { pCreate->blocksPerMeter = htons(pCreate->blocksPerMeter); pCreate->rowsInFileBlock = htonl(pCreate->rowsInFileBlock); - if (!pConn->writeAuth) { + if (grantCheckExpired()) { + code = TSDB_CODE_GRANT_EXPIRED; + } else if (!pConn->writeAuth) { code = TSDB_CODE_NO_RIGHTS; } else { - SAcctObj *pAcct = &acctObj; - code = mgmtCreateDb(pAcct, pCreate); + code = mgmtCreateDb(pConn->pAcct, pCreate); if (code == TSDB_CODE_SUCCESS) { mLPrint("DB:%s is created by %s", pCreate->db, pConn->pUser->user); } @@ -363,17 +569,26 @@ int mgmtProcessCreateDbMsg(char *pMsg, int msgLen, SConnObj *pConn) { return 0; } +int mgmtProcessCreateMnodeMsg(char *pMsg, int msgLen, SConnObj *pConn) { + return taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_CREATE_MNODE_RSP, TSDB_CODE_OPS_NOT_SUPPORT); +} + int mgmtProcessAlterDbMsg(char *pMsg, int msgLen, SConnObj *pConn) { SAlterDbMsg *pAlter = (SAlterDbMsg *)pMsg; int code = 0; + if (mgmtCheckRedirectMsg(pConn, TSDB_MSG_TYPE_ALTER_DB_RSP) != 0) { + return 0; + } + pAlter->daysPerFile = htonl(pAlter->daysPerFile); pAlter->daysToKeep = htonl(pAlter->daysToKeep); + pAlter->maxSessions = htonl(pAlter->maxSessions) + 1; if (!pConn->writeAuth) { code = TSDB_CODE_NO_RIGHTS; } else { - code = mgmtAlterDb(&acctObj, pAlter); + code = mgmtAlterDb(pConn->pAcct, pAlter); if (code == TSDB_CODE_SUCCESS) { mLPrint("DB:%s is altered by %s", pAlter->db, pConn->pUser->user); } @@ -384,18 +599,6 @@ int mgmtProcessAlterDbMsg(char *pMsg, int msgLen, SConnObj *pConn) { return 0; } -int mgmtProcessCfgDnodeMsg(char *pMsg, int msgLen, SConnObj *pConn) { - SCfgMsg *pCfg = (SCfgMsg *)pMsg; - - int code = tsCfgDynamicOptions(pCfg->config); - - taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_CFG_PNODE_RSP, code); - - if (code == 0) mTrace("dnode is configured by %s", pConn->pUser->user); - - return 0; -} - int mgmtProcessKillQueryMsg(char *pMsg, int msgLen, SConnObj *pConn) { int code = 0; SKillQuery *pKill = (SKillQuery *)pMsg; @@ -445,8 +648,12 @@ int mgmtProcessCreateUserMsg(char *pMsg, int msgLen, SConnObj *pConn) { SCreateUserMsg *pCreate = (SCreateUserMsg *)pMsg; int code = 0; + if (mgmtCheckRedirectMsg(pConn, TSDB_MSG_TYPE_CREATE_USER_RSP) != 0) { + return 0; + } + if (pConn->superAuth) { - code = mgmtCreateUser(&acctObj, pCreate->user, pCreate->pass); + code = mgmtCreateUser(pConn->pAcct, pCreate->user, pCreate->pass); if (code == TSDB_CODE_SUCCESS) { mLPrint("user:%s is created by %s", pCreate->user, pConn->pUser->user); } @@ -464,6 +671,10 @@ int mgmtProcessAlterUserMsg(char *pMsg, int msgLen, SConnObj *pConn) { int code = 0; SUserObj * pUser; + if (mgmtCheckRedirectMsg(pConn, TSDB_MSG_TYPE_ALTER_USER_RSP) != 0) { + return 0; + } + pUser = mgmtGetUser(pAlter->user); if (pUser == NULL) { taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_ALTER_USER_RSP, TSDB_CODE_INVALID_USER); @@ -473,11 +684,11 @@ int mgmtProcessAlterUserMsg(char *pMsg, int msgLen, SConnObj *pConn) { if (strcmp(pUser->user, "monitor") == 0 || strcmp(pUser->user, "stream") == 0) { code = TSDB_CODE_NO_RIGHTS; } else if ((strcmp(pUser->user, pConn->pUser->user) == 0) || - ((strcmp(pUser->acct, acctObj.user) == 0) && pConn->superAuth) || + ((strcmp(pUser->acct, pConn->pAcct->user) == 0) && pConn->superAuth) || (strcmp(pConn->pUser->user, "root") == 0)) { if ((pAlter->flag & TSDB_ALTER_USER_PASSWD) != 0) { memset(pUser->pass, 0, sizeof(pUser->pass)); - taosEncryptPass((uint8_t *)(pAlter->pass), strlen(pAlter->pass), pUser->pass); + taosEncryptPass(pAlter->pass, strlen(pAlter->pass), pUser->pass); } if ((pAlter->flag & TSDB_ALTER_USER_PRIVILEGES) != 0) { if (pAlter->privilege == 1) { // super @@ -509,13 +720,17 @@ int mgmtProcessDropUserMsg(char *pMsg, int msgLen, SConnObj *pConn) { SDropUserMsg *pDrop = (SDropUserMsg *)pMsg; int code = 0; + if (mgmtCheckRedirectMsg(pConn, TSDB_MSG_TYPE_DROP_USER_RSP) != 0) { + return 0; + } + if (strcmp(pConn->pUser->user, pDrop->user) == 0) { code = TSDB_CODE_NO_RIGHTS; } else if (strcmp(pDrop->user, "monitor") == 0 || strcmp(pDrop->user, "stream") == 0) { code = TSDB_CODE_NO_RIGHTS; } else { if (pConn->superAuth) { - code = mgmtDropUser(&acctObj, pDrop->user); + code = mgmtDropUser(pConn->pAcct, pDrop->user); if (code == 0) { mLPrint("user:%s is dropped by %s", pDrop->user, pConn->pUser->user); } @@ -533,10 +748,14 @@ int mgmtProcessDropDbMsg(char *pMsg, int msgLen, SConnObj *pConn) { SDropDbMsg *pDrop = (SDropDbMsg *)pMsg; int code; + if (mgmtCheckRedirectMsg(pConn, TSDB_MSG_TYPE_DROP_DB_RSP) != 0) { + return 0; + } + if (!pConn->writeAuth) { code = TSDB_CODE_NO_RIGHTS; } else { - code = mgmtDropDbByName(&acctObj, pDrop->db); + code = mgmtDropDbByName(pConn->pAcct, pDrop->db); if (code == 0) { mLPrint("DB:%s is dropped by %s", pDrop->db, pConn->pUser->user); } @@ -559,13 +778,15 @@ int mgmtProcessUseDbMsg(char *pMsg, int msgLen, SConnObj *pConn) { } int (*mgmtGetMetaFp[])(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) = { - mgmtGetUserMeta, mgmtGetDbMeta, mgmtGetMeterMeta, mgmtGetDnodeMeta, mgmtGetVgroupMeta, - mgmtGetMetricMeta, mgmtGetQueryMeta, mgmtGetStreamMeta, mgmtGetConnsMeta, + mgmtGetAcctMeta, mgmtGetUserMeta, mgmtGetDbMeta, mgmtGetMeterMeta, mgmtGetDnodeMeta, + mgmtGetMnodeMeta, mgmtGetVgroupMeta, mgmtGetMetricMeta, mgmtGetModuleMeta, mgmtGetQueryMeta, + mgmtGetStreamMeta, mgmtGetConfigMeta, mgmtGetConnsMeta, mgmtGetScoresMeta, grantGetGrantsMeta, }; int (*mgmtRetrieveFp[])(SShowObj *pShow, char *data, int rows, SConnObj *pConn) = { - mgmtRetrieveUsers, mgmtRetrieveDbs, mgmtRetrieveMeters, mgmtRetrieveDnodes, mgmtRetrieveVgroups, - mgmtRetrieveMetrics, mgmtRetrieveQueries, mgmtRetrieveStreams, mgmtRetrieveConns, + mgmtRetrieveAccts, mgmtRetrieveUsers, mgmtRetrieveDbs, mgmtRetrieveMeters, mgmtRetrieveDnodes, + mgmtRetrieveMnodes, mgmtRetrieveVgroups, mgmtRetrieveMetrics, mgmtRetrieveModules, mgmtRetrieveQueries, + mgmtRetrieveStreams, mgmtRetrieveConfigs, mgmtRetrieveConns, mgmtRetrieveScores, grantRetrieveGrants, }; int mgmtProcessShowMsg(char *pMsg, int msgLen, SConnObj *pConn) { @@ -576,10 +797,20 @@ int mgmtProcessShowMsg(char *pMsg, int msgLen, SConnObj *pConn) { SShowRspMsg *pShowRsp; SShowObj * pShow = NULL; + if (pShowMsg->type == TSDB_MGMT_TABLE_PNODE || TSDB_MGMT_TABLE_GRANTS || TSDB_MGMT_TABLE_SCORES) { + if (mgmtCheckRedirectMsg(pConn, TSDB_MSG_TYPE_SHOW_RSP) != 0) { + return 0; + } + } + int size = sizeof(STaosHeader) + sizeof(STaosRsp) + sizeof(SShowRspMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE; pStart = taosBuildRspMsgWithSize(pConn->thandle, TSDB_MSG_TYPE_SHOW_RSP, size); - if (pStart == NULL) return 0; + if (pStart == NULL) { + taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_SHOW_RSP, TSDB_CODE_SERV_OUT_OF_MEMORY); + return 0; + } + pMsg = pStart; pRsp = (STaosRsp *)pMsg; pMsg = (char *)pRsp->more; @@ -604,7 +835,7 @@ int mgmtProcessShowMsg(char *pMsg, int msgLen, SConnObj *pConn) { if (code == 0) { pMsg += sizeof(SShowRspMsg) + sizeof(SSchema) * pShow->numOfColumns; } else { - mError("pShow:%p, failed to get Meta, code:%d", pShow, code); + mError("pShow:%p, type:%d %s, failed to get Meta, code:%d", pShow, pShowMsg->type, taosMsg[pShowMsg->type], code); free(pShow); } } @@ -627,9 +858,9 @@ int mgmtProcessRetrieveMsg(char *pMsg, int msgLen, SConnObj *pConn) { pRetrieve = (SRetrieveMeterMsg *)pMsg; /* - * in case of server restart, apps may hold qhandle created by server before restart, - * which is actually invalid, therefore, signature check is required. - */ + * in case of server restart, apps may hold qhandle created by server before + * restart, which is actually invalid, therefore, signature check is required. + */ if (pRetrieve->qhandle != pConn->qhandle) { mError("retrieve:%p, qhandle:%p is not matched with saved:%p", pRetrieve, pRetrieve->qhandle, pConn->qhandle); taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_RETRIEVE_RSP, TSDB_CODE_MEMORY_CORRUPTED); @@ -642,7 +873,9 @@ int mgmtProcessRetrieveMsg(char *pMsg, int msgLen, SConnObj *pConn) { taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_RETRIEVE_RSP, TSDB_CODE_MEMORY_CORRUPTED); return -1; } else { - if (pRetrieve->free == 0) rowsToRead = pShow->numOfRows - pShow->numOfReads; + if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) == 0) { + rowsToRead = pShow->numOfRows - pShow->numOfReads; + } /* return no more than 100 meters in one round trip */ if (rowsToRead > 100) rowsToRead = 100; @@ -656,7 +889,11 @@ int mgmtProcessRetrieveMsg(char *pMsg, int msgLen, SConnObj *pConn) { } pStart = taosBuildRspMsgWithSize(pConn->thandle, TSDB_MSG_TYPE_RETRIEVE_RSP, size + 100); - if (pStart == NULL) return 0; + if (pStart == NULL) { + taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_RETRIEVE_RSP, TSDB_CODE_SERV_OUT_OF_MEMORY); + return 0; + } + pMsg = pStart; STaosRsp *pTaosRsp = (STaosRsp *)pStart; @@ -668,7 +905,8 @@ int mgmtProcessRetrieveMsg(char *pMsg, int msgLen, SConnObj *pConn) { pMsg = pRsp->data; // if free flag is set, client wants to clean the resources - if (pRetrieve->free == 0) rowsRead = (*mgmtRetrieveFp[pShow->type])(pShow, pRsp->data, rowsToRead, pConn); + if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) == 0) + rowsRead = (*mgmtRetrieveFp[pShow->type])(pShow, pRsp->data, rowsToRead, pConn); if (rowsRead < 0) { rowsRead = 0; @@ -701,6 +939,10 @@ int mgmtProcessCreateTableMsg(char *pMsg, int msgLen, SConnObj *pConn) { int code; SSchema * pSchema; + if (mgmtCheckRedirectMsg(pConn, TSDB_MSG_TYPE_CREATE_TABLE_RSP) != 0) { + return 0; + } + if (!pConn->writeAuth) { code = TSDB_CODE_NO_RIGHTS; } else { @@ -735,6 +977,10 @@ int mgmtProcessDropTableMsg(char *pMsg, int msgLen, SConnObj *pConn) { SDropTableMsg *pDrop = (SDropTableMsg *)pMsg; int code; + if (mgmtCheckRedirectMsg(pConn, TSDB_MSG_TYPE_DROP_TABLE_RSP) != 0) { + return 0; + } + if (!pConn->writeAuth) { code = TSDB_CODE_NO_RIGHTS; } else { @@ -754,6 +1000,10 @@ int mgmtProcessAlterTableMsg(char *pMsg, int msgLen, SConnObj *pConn) { SAlterTableMsg *pAlter = (SAlterTableMsg *)pMsg; int code; + if (mgmtCheckRedirectMsg(pConn, TSDB_MSG_TYPE_ALTER_TABLE_RSP) != 0) { + return 0; + } + if (!pConn->writeAuth) { code = TSDB_CODE_NO_RIGHTS; } else { @@ -784,6 +1034,27 @@ int mgmtProcessAlterTableMsg(char *pMsg, int msgLen, SConnObj *pConn) { return 0; } +int mgmtProcessCfgDnodeMsg(char *pMsg, int msgLen, SConnObj *pConn) { + int code = 0; + SCfgMsg *pCfg = (SCfgMsg *)pMsg; + + if (mgmtCheckRedirectMsg(pConn, TSDB_MSG_TYPE_CFG_MNODE_RSP) != 0) { + return 0; + } + + if (strcmp(pConn->pAcct->user, "root") != 0) { + code = TSDB_CODE_NO_RIGHTS; + } else { + code = mgmtSendCfgDnodeMsg(pMsg); + } + + taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_CFG_PNODE_RSP, code); + + if (code == 0) mTrace("dnode:%s is configured by %s", pCfg->ip, pConn->pUser->user); + + return 0; +} + int mgmtProcessHeartBeatMsg(char *cont, int contLen, SConnObj *pConn) { char * pStart, *pMsg; int msgLen; @@ -806,8 +1077,14 @@ int mgmtProcessHeartBeatMsg(char *cont, int contLen, SConnObj *pConn) { pConn->streamId = 0; pHBRsp->killConnection = pConn->killConnection; +#ifdef CLUSTER + int size = pSdbPublicIpList->numOfIps * 4; + pHBRsp->ipList.numOfIps = pSdbPublicIpList->numOfIps; + memcpy(pHBRsp->ipList.ip, pSdbPublicIpList->ip, size); + pMsg += sizeof(SHeartBeatRsp) + size; +#else pMsg += sizeof(SHeartBeatRsp); - +#endif msgLen = pMsg - pStart; taosSendMsgToPeer(pConn->thandle, pStart, msgLen); @@ -817,9 +1094,10 @@ int mgmtProcessHeartBeatMsg(char *cont, int contLen, SConnObj *pConn) { void mgmtEstablishConn(SConnObj *pConn) { __sync_fetch_and_add(&mgmtShellConns, 1); + __sync_fetch_and_add(&sdbExtConns, 1); pConn->stime = taosGetTimestampMs(); - if (strcmp(pConn->pUser->user, "root") == 0 || strcmp(pConn->pUser->user, acctObj.user) == 0) { + if (strcmp(pConn->pUser->user, "root") == 0 || strcmp(pConn->pUser->user, pConn->pAcct->user) == 0) { pConn->superAuth = 1; pConn->writeAuth = 1; } else { @@ -872,7 +1150,12 @@ int mgmtProcessConnectMsg(char *pMsg, int msgLen, SConnObj *pConn) { goto _rsp; } - pAcct = &acctObj; + if (grantCheckExpired()) { + code = TSDB_CODE_GRANT_EXPIRED; + goto _rsp; + } + + pAcct = mgmtGetAcct(pUser->acct); if (pConnectMsg->db[0]) { memset(dbName, 0, sizeof(dbName)); @@ -887,6 +1170,7 @@ int mgmtProcessConnectMsg(char *pMsg, int msgLen, SConnObj *pConn) { if (pConn->pAcct) { mgmtRemoveConnFromAcct(pConn); __sync_fetch_and_sub(&mgmtShellConns, 1); + __sync_fetch_and_sub(&sdbExtConns, 1); } code = 0; @@ -912,6 +1196,12 @@ _rsp: pConnectRsp->superAuth = pConn->superAuth; pMsg += sizeof(SConnectRsp); +#ifdef CLUSTER + int size = pSdbPublicIpList->numOfIps * 4 + sizeof(SIpList); + memcpy(pMsg, pSdbPublicIpList, size); + pMsg += size; +#endif + // set the time resolution: millisecond or microsecond *((uint32_t *)pMsg) = tsTimePrecision; pMsg += sizeof(uint32_t); @@ -939,6 +1229,7 @@ void *mgmtProcessMsgFromShell(char *msg, void *ahandle, void *thandle) { if (pConn) { mgmtRemoveConnFromAcct(pConn); __sync_fetch_and_sub(&mgmtShellConns, 1); + __sync_fetch_and_sub(&sdbExtConns, 1); mTrace("connection from %s is closed", pConn->pUser->user); memset(pConn, 0, sizeof(SConnObj)); } @@ -946,6 +1237,13 @@ void *mgmtProcessMsgFromShell(char *msg, void *ahandle, void *thandle) { return NULL; } +#ifdef CLUSTER + if (sdbInited == NULL || sdbStatus != SDB_STATUS_SERVING) { + taosSendSimpleRsp(thandle, pMsg->msgType + 1, TSDB_CODE_NOT_READY); + mTrace("shell msg is ignored since SDB is not ready"); + } +#endif + if (pConn == NULL) { pConn = connList + pMsg->destId; pConn->thandle = thandle; @@ -959,7 +1257,7 @@ void *mgmtProcessMsgFromShell(char *msg, void *ahandle, void *thandle) { if (pConn->pAcct == NULL) { pConn->pUser = mgmtGetUser(pConn->user); if (pConn->pUser) { - pConn->pAcct = &acctObj; + pConn->pAcct = mgmtGetAcct(pConn->pUser->acct); mgmtEstablishConn(pConn); mTrace("login from:%x:%d", pConn->ip, htons(pConn->port)); } @@ -977,7 +1275,7 @@ void *mgmtProcessMsgFromShell(char *msg, void *ahandle, void *thandle) { // read-only request can be executed concurrently if ((pMsg->msgType == TSDB_MSG_TYPE_METERINFO && (!mgmtCheckMeterMetaMsgType(cont))) || pMsg->msgType == TSDB_MSG_TYPE_METRIC_META || pMsg->msgType == TSDB_MSG_TYPE_RETRIEVE || - pMsg->msgType == TSDB_MSG_TYPE_SHOW) { + pMsg->msgType == TSDB_MSG_TYPE_SHOW || pMsg->msgType == TSDB_MSG_TYPE_MULTI_METERINFO) { (*mgmtProcessShellMsg[pMsg->msgType])(cont, contLen, pConn); } else { if (mgmtProcessShellMsg[pMsg->msgType]) { @@ -1011,12 +1309,16 @@ void *mgmtProcessMsgFromShell(char *msg, void *ahandle, void *thandle) { void mgmtInitProcessShellMsg() { mgmtProcessShellMsg[TSDB_MSG_TYPE_METERINFO] = mgmtProcessMeterMetaMsg; mgmtProcessShellMsg[TSDB_MSG_TYPE_METRIC_META] = mgmtProcessMetricMetaMsg; + mgmtProcessShellMsg[TSDB_MSG_TYPE_MULTI_METERINFO] = mgmtProcessMultiMeterMetaMsg; mgmtProcessShellMsg[TSDB_MSG_TYPE_CREATE_DB] = mgmtProcessCreateDbMsg; mgmtProcessShellMsg[TSDB_MSG_TYPE_ALTER_DB] = mgmtProcessAlterDbMsg; mgmtProcessShellMsg[TSDB_MSG_TYPE_CREATE_USER] = mgmtProcessCreateUserMsg; mgmtProcessShellMsg[TSDB_MSG_TYPE_ALTER_USER] = mgmtProcessAlterUserMsg; + mgmtProcessShellMsg[TSDB_MSG_TYPE_CREATE_ACCT] = mgmtProcessCreateAcctMsg; mgmtProcessShellMsg[TSDB_MSG_TYPE_DROP_DB] = mgmtProcessDropDbMsg; mgmtProcessShellMsg[TSDB_MSG_TYPE_DROP_USER] = mgmtProcessDropUserMsg; + mgmtProcessShellMsg[TSDB_MSG_TYPE_DROP_ACCT] = mgmtProcessDropAcctMsg; + mgmtProcessShellMsg[TSDB_MSG_TYPE_ALTER_ACCT] = mgmtProcessAlterAcctMsg; mgmtProcessShellMsg[TSDB_MSG_TYPE_CREATE_TABLE] = mgmtProcessCreateTableMsg; mgmtProcessShellMsg[TSDB_MSG_TYPE_DROP_TABLE] = mgmtProcessDropTableMsg; @@ -1027,8 +1329,15 @@ void mgmtInitProcessShellMsg() { mgmtProcessShellMsg[TSDB_MSG_TYPE_SHOW] = mgmtProcessShowMsg; mgmtProcessShellMsg[TSDB_MSG_TYPE_CONNECT] = mgmtProcessConnectMsg; mgmtProcessShellMsg[TSDB_MSG_TYPE_HEARTBEAT] = mgmtProcessHeartBeatMsg; + mgmtProcessShellMsg[TSDB_MSG_TYPE_CREATE_PNODE] = mgmtProcessCreateDnodeMsg; + mgmtProcessShellMsg[TSDB_MSG_TYPE_DROP_PNODE] = mgmtProcessDropDnodeMsg; + mgmtProcessShellMsg[TSDB_MSG_TYPE_CREATE_MNODE] = mgmtProcessCreateMnodeMsg; + mgmtProcessShellMsg[TSDB_MSG_TYPE_DROP_MNODE] = mgmtProcessDropMnodeMsg; + mgmtProcessShellMsg[TSDB_MSG_TYPE_CFG_MNODE] = mgmtProcessCfgMnodeMsg; mgmtProcessShellMsg[TSDB_MSG_TYPE_CFG_PNODE] = mgmtProcessCfgDnodeMsg; mgmtProcessShellMsg[TSDB_MSG_TYPE_KILL_QUERY] = mgmtProcessKillQueryMsg; mgmtProcessShellMsg[TSDB_MSG_TYPE_KILL_STREAM] = mgmtProcessKillStreamMsg; mgmtProcessShellMsg[TSDB_MSG_TYPE_KILL_CONNECTION] = mgmtProcessKillConnectionMsg; } + +#pragma GCC diagnostic pop diff --git a/src/system/detail/src/mgmtSupertableQuery.c b/src/system/detail/src/mgmtSupertableQuery.c new file mode 100644 index 0000000000..31f3e29425 --- /dev/null +++ b/src/system/detail/src/mgmtSupertableQuery.c @@ -0,0 +1,842 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include +#include +#include +#include + +#include "mgmt.h" +#include "mgmtUtil.h" +#include "textbuffer.h" +#include "tschemautil.h" +#include "tsqlfunction.h" +#include "vnodeTagMgmt.h" + +typedef struct SJoinSupporter { + void** val; + void** pTabObjs; + int32_t size; + + int16_t type; + int16_t colIndex; + + void** qualMeterObj; + int32_t qualSize; +} SJoinSupporter; + +typedef struct SMeterNameFilterSupporter { + SPatternCompareInfo info; + char* pattern; +} SMeterNameFilterSupporter; + +static void tansformQueryResult(tQueryResultset* pRes); + +static int32_t tabObjVGIDComparator(const void* pLeft, const void* pRight) { + STabObj* p1 = *(STabObj**)pLeft; + STabObj* p2 = *(STabObj**)pRight; + + int32_t ret = p1->gid.vgId - p2->gid.vgId; + if (ret == 0) { + return ret; + } else { + return ret > 0 ? 1 : -1; + } +} + +// monotonic inc in memory address +static int32_t tabObjPointerComparator(const void* pLeft, const void* pRight) { + int64_t ret = (int64_t)pLeft - (int64_t)pRight; + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +static int32_t tabObjResultComparator(const void* p1, const void* p2, void* param) { + tOrderDescriptor* pOrderDesc = (tOrderDescriptor*)param; + + STabObj* pNode1 = (STabObj*)p1; + STabObj* pNode2 = (STabObj*)p2; + + for (int32_t i = 0; i < pOrderDesc->orderIdx.numOfOrderedCols; ++i) { + int32_t colIdx = pOrderDesc->orderIdx.pData[i]; + + char* f1 = NULL; + char* f2 = NULL; + + SSchema schema = {0}; + + if (colIdx == -1) { + f1 = pNode1->meterId; + f2 = pNode2->meterId; + schema.type = TSDB_DATA_TYPE_BINARY; + schema.bytes = TSDB_METER_ID_LEN; + } else { + f1 = mgmtMeterGetTag(pNode1, colIdx, NULL); + f2 = mgmtMeterGetTag(pNode2, colIdx, &schema); + assert(schema.type == pOrderDesc->pTagSchema->pSchema[colIdx].type); + } + + int32_t ret = doCompare(f1, f2, schema.type, schema.bytes); + if (ret == 0) { + continue; + } else { + return ret; + } + } + + return 0; +} + +// todo merge sort function with losertree used +void mgmtReorganizeMetersInMetricMeta(SMetricMetaMsg* pMetricMetaMsg, int32_t tableIndex, tQueryResultset* pRes) { + if (pRes->num <= 0) { // no result, no need to pagination + return; + } + + SMetricMetaElemMsg* pElem = (SMetricMetaElemMsg*)((char*)pMetricMetaMsg + pMetricMetaMsg->metaElem[tableIndex]); + + STabObj* pMetric = mgmtGetMeter(pElem->meterId); + SSchema* pTagSchema = (SSchema*)(pMetric->schema + pMetric->numOfColumns * sizeof(SSchema)); + + /* + * To apply the group limitation and group offset, we should sort the result + * list according to the order condition + */ + tOrderDescriptor* descriptor = + (tOrderDescriptor*)calloc(1, sizeof(tOrderDescriptor) + sizeof(int32_t) * pElem->numOfGroupCols); + descriptor->pTagSchema = tCreateTagSchema(pTagSchema, pMetric->numOfTags); + descriptor->orderIdx.numOfOrderedCols = pElem->numOfGroupCols; + + int32_t* startPos = NULL; + int32_t numOfSubset = 1; + + if (pElem->numOfGroupCols > 0) { + SColIndexEx* groupColumnList = (SColIndexEx*)((char*)pMetricMetaMsg + pElem->groupbyTagColumnList); + for (int32_t i = 0; i < pElem->numOfGroupCols; ++i) { + descriptor->orderIdx.pData[i] = groupColumnList[i].colIdx; + } + + tQSortEx(pRes->pRes, POINTER_BYTES, 0, pRes->num - 1, descriptor, tabObjResultComparator); + startPos = calculateSubGroup(pRes->pRes, pRes->num, &numOfSubset, descriptor, tabObjResultComparator); + } else { + startPos = malloc(2 * sizeof(int32_t)); + + startPos[0] = 0; + startPos[1] = (int32_t)pRes->num; + } + + /* + * sort the result according to vgid to ensure meters with the same vgid is + * continuous in the result list + */ + qsort(pRes->pRes, (size_t)pRes->num, POINTER_BYTES, tabObjVGIDComparator); + + free(descriptor->pTagSchema); + free(descriptor); + free(startPos); +} + +static void mgmtRetrieveByMeterName(tQueryResultset* pRes, char* str, STabObj* pMetric) { + const char* sep = ","; + char* pToken = NULL; + + int32_t s = 4; // initial size + + pRes->pRes = malloc(sizeof(char*) * s); + pRes->num = 0; + + for (pToken = strsep(&str, sep); pToken != NULL; pToken = strsep(&str, sep)) { + STabObj* pMeterObj = mgmtGetMeter(pToken); + if (pMeterObj == NULL) { + mWarn("metric:%s error in metric query expression, invalid meter id:%s", pMetric->meterId, pToken); + continue; + } + + if (pRes->num >= s) { + s += (s >> 1); // increase 50% size + pRes->pRes = realloc(pRes->pRes, sizeof(char*) * s); + } + + /* not a table created from metric, ignore */ + if (pMeterObj->meterType != TSDB_METER_MTABLE) { + continue; + } + + /* + * queried meter not belongs to this metric, ignore, metric does not have + * uid, so compare according to meterid + */ + STabObj* parentMetric = mgmtGetMeter(pMeterObj->pTagData); + if (strncasecmp(parentMetric->meterId, pMetric->meterId, TSDB_METER_ID_LEN) != 0 || + (parentMetric->uid != pMetric->uid)) { + continue; + } + + pRes->pRes[pRes->num++] = pMeterObj; + } +} + +static bool mgmtTablenameFilterCallback(tSkipListNode* pNode, void* param) { + SMeterNameFilterSupporter* pSupporter = (SMeterNameFilterSupporter*)param; + + char name[TSDB_METER_ID_LEN] = {0}; + + // pattern compare for meter name + STabObj* pMeterObj = (STabObj*)pNode->pData; + extractMeterName(pMeterObj->meterId, name); + + return patternMatch(pSupporter->pattern, name, TSDB_METER_ID_LEN, &pSupporter->info) == TSDB_PATTERN_MATCH; +} + +static void mgmtRetrieveFromLikeOptr(tQueryResultset* pRes, const char* str, STabObj* pMetric) { + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + SMeterNameFilterSupporter supporter = {info, str}; + + pRes->num = + tSkipListIterateList(pMetric->pSkipList, (tSkipListNode***)&pRes->pRes, mgmtTablenameFilterCallback, &supporter); +} + +static void mgmtFilterByTableNameCond(tQueryResultset* pRes, char* condStr, int32_t len, STabObj* pMetric) { + pRes->num = 0; + if (len <= 0) { + return; + } + + char* str = calloc(1, (size_t)len + 1); + memcpy(str, condStr, len); + + if (strncasecmp(condStr, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) { // handle in expression + mgmtRetrieveByMeterName(pRes, str + QUERY_COND_REL_PREFIX_IN_LEN, pMetric); + } else { // handle like expression + assert(strncasecmp(str, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0); + mgmtRetrieveFromLikeOptr(pRes, str + QUERY_COND_REL_PREFIX_LIKE_LEN, pMetric); + + tansformQueryResult(pRes); + } + + free(str); +} + +static bool mgmtJoinFilterCallback(tSkipListNode* pNode, void* param) { + SJoinSupporter* pSupporter = (SJoinSupporter*)param; + + SSchema s = {0}; + char* v = mgmtMeterGetTag((STabObj*)pNode->pData, pSupporter->colIndex, &s); + + for (int32_t i = 0; i < pSupporter->size; ++i) { + int32_t ret = doCompare(v, pSupporter->val[i], pSupporter->type, s.bytes); + if (ret == 0) { + pSupporter->qualMeterObj[pSupporter->qualSize++] = pSupporter->pTabObjs[i]; + + /* + * Once a value is qualified according to the join condition, it is remove from the + * candidate list, as well as its corresponding meter object. + * + * last one does not need to move forward + */ + if (i < pSupporter->size - 1) { + memmove(pSupporter->val[i], pSupporter->val[i + 1], pSupporter->size - (i + 1)); + } + + pSupporter->size -= 1; + + return true; + } + } + + return false; +} + +static void orderResult(SMetricMetaMsg* pMetricMetaMsg, tQueryResultset* pRes, int16_t colIndex, int32_t tableIndex) { + SMetricMetaElemMsg* pElem = (SMetricMetaElemMsg*)((char*)pMetricMetaMsg + pMetricMetaMsg->metaElem[tableIndex]); + + tOrderDescriptor* descriptor = + (tOrderDescriptor*)calloc(1, sizeof(tOrderDescriptor) + sizeof(int32_t) * 1); // only one column for join + + STabObj* pMetric = mgmtGetMeter(pElem->meterId); + SSchema* pTagSchema = (SSchema*)(pMetric->schema + pMetric->numOfColumns * sizeof(SSchema)); + + descriptor->pTagSchema = tCreateTagSchema(pTagSchema, pMetric->numOfTags); + + descriptor->orderIdx.pData[0] = colIndex; + descriptor->orderIdx.numOfOrderedCols = 1; + + // sort results list + tQSortEx(pRes->pRes, POINTER_BYTES, 0, pRes->num - 1, descriptor, tabObjResultComparator); + + free(descriptor->pTagSchema); + free(descriptor); +} + +// check for duplicate join tags +static int32_t mgmtCheckForDuplicateTagValue(tQueryResultset* pRes, int32_t index, int32_t tagCol) { + SSchema s = {0}; + + for (int32_t k = 1; k < pRes[index].num; ++k) { + STabObj* pObj1 = pRes[index].pRes[k - 1]; + STabObj* pObj2 = pRes[index].pRes[k]; + + char* val1 = mgmtMeterGetTag(pObj1, tagCol, &s); + char* val2 = mgmtMeterGetTag(pObj2, tagCol, NULL); + + if (doCompare(val1, val2, s.type, s.bytes) == 0) { + return TSDB_CODE_DUPLICATE_TAGS; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t mgmtDoJoin(SMetricMetaMsg* pMetricMetaMsg, tQueryResultset* pRes) { + if (pMetricMetaMsg->numOfMeters == 1) { + return TSDB_CODE_SUCCESS; + } + + bool allEmpty = false; + for (int32_t i = 0; i < pMetricMetaMsg->numOfMeters; ++i) { + if (pRes->num == 0) { // all results are empty if one of them is empty + allEmpty = true; + break; + } + } + + if (allEmpty) { + for (int32_t i = 0; i < pMetricMetaMsg->numOfMeters; ++i) { + pRes[i].num = 0; + tfree(pRes[i].pRes); + } + + return TSDB_CODE_SUCCESS; + } + + char* cond = (char*)pMetricMetaMsg + pMetricMetaMsg->join; + + char left[TSDB_METER_ID_LEN + 1] = {0}; + strcpy(left, cond); + int16_t leftTagColIndex = *(int16_t*)(cond + TSDB_METER_ID_LEN); + + char right[TSDB_METER_ID_LEN + 1] = {0}; + strcpy(right, cond + TSDB_METER_ID_LEN + sizeof(int16_t)); + int16_t rightTagColIndex = *(int16_t*)(cond + TSDB_METER_ID_LEN * 2 + sizeof(int16_t)); + + STabObj* pLeftMetric = mgmtGetMeter(left); + STabObj* pRightMetric = mgmtGetMeter(right); + + // decide the pRes belongs to + int32_t leftIndex = 0; + int32_t rightIndex = 0; + + for (int32_t i = 0; i < pMetricMetaMsg->numOfMeters; ++i) { + STabObj* pObj = (STabObj*)pRes[i].pRes[0]; + STabObj* pMetric1 = mgmtGetMeter(pObj->pTagData); + if (pMetric1 == pLeftMetric) { + leftIndex = i; + } else if (pMetric1 == pRightMetric) { + rightIndex = i; + } + } + + orderResult(pMetricMetaMsg, &pRes[leftIndex], leftTagColIndex, leftIndex); + orderResult(pMetricMetaMsg, &pRes[rightIndex], rightTagColIndex, rightIndex); + + int32_t i = 0; + int32_t j = 0; + + SSchema s = {0}; + int32_t res = 0; + + // check for duplicated tag values + int32_t ret1 = mgmtCheckForDuplicateTagValue(pRes, leftIndex, leftTagColIndex); + int32_t ret2 = mgmtCheckForDuplicateTagValue(pRes, rightIndex, rightTagColIndex); + if (ret1 != TSDB_CODE_SUCCESS || ret2 != TSDB_CODE_SUCCESS) { + return ret1; + } + + while (i < pRes[leftIndex].num && j < pRes[rightIndex].num) { + STabObj* pLeftObj = pRes[leftIndex].pRes[i]; + STabObj* pRightObj = pRes[rightIndex].pRes[j]; + + char* v1 = mgmtMeterGetTag(pLeftObj, leftTagColIndex, &s); + char* v2 = mgmtMeterGetTag(pRightObj, rightTagColIndex, NULL); + + int32_t ret = doCompare(v1, v2, s.type, s.bytes); + if (ret == 0) { // qualified + pRes[leftIndex].pRes[res] = pRes[leftIndex].pRes[i++]; + pRes[rightIndex].pRes[res] = pRes[rightIndex].pRes[j++]; + + res++; + } else if (ret < 0) { + i++; + } else { + j++; + } + } + + pRes[leftIndex].num = res; + pRes[rightIndex].num = res; + + return TSDB_CODE_SUCCESS; +} + +/** + * convert the result pointer to STabObj instead of tSkipListNode + * @param pRes + */ +static void tansformQueryResult(tQueryResultset* pRes) { + if (pRes == NULL || pRes->num == 0) { + return; + } + + for (int32_t i = 0; i < pRes->num; ++i) { + pRes->pRes[i] = ((tSkipListNode*)(pRes->pRes[i]))->pData; + } +} + +static tQueryResultset* doNestedLoopIntersect(tQueryResultset* pRes1, tQueryResultset* pRes2) { + int32_t num = 0; + void** pResult = pRes1->pRes; + + for (int32_t i = 0; i < pRes1->num; ++i) { + for (int32_t j = 0; j < pRes2->num; ++j) { + if (pRes1->pRes[i] == pRes2->pRes[j]) { + pResult[num++] = pRes1->pRes[i]; + break; + } + } + } + + tQueryResultClean(pRes2); + + memset(pRes1->pRes + num, 0, sizeof(void*) * (pRes1->num - num)); + pRes1->num = num; + + return pRes1; +} + +static tQueryResultset* doSortIntersect(tQueryResultset* pRes1, tQueryResultset* pRes2) { + size_t sizePtr = sizeof(void*); + + qsort(pRes1->pRes, pRes1->num, sizePtr, tabObjPointerComparator); + qsort(pRes2->pRes, pRes2->num, sizePtr, tabObjPointerComparator); + + int32_t i = 0; + int32_t j = 0; + + int32_t num = 0; + while (i < pRes1->num && j < pRes2->num) { + if (pRes1->pRes[i] == pRes2->pRes[j]) { + j++; + pRes1->pRes[num++] = pRes1->pRes[i++]; + } else if (pRes1->pRes[i] < pRes2->pRes[j]) { + i++; + } else { + j++; + } + } + + tQueryResultClean(pRes2); + + memset(pRes1->pRes + num, 0, sizeof(void*) * (pRes1->num - num)); + pRes1->num = num; + return pRes1; +} + +static void queryResultIntersect(tQueryResultset* pFinalRes, tQueryResultset* pRes) { + const int32_t NUM_OF_RES_THRESHOLD = 20; + + // for small result, use nested loop join + if (pFinalRes->num <= NUM_OF_RES_THRESHOLD && pRes->num <= NUM_OF_RES_THRESHOLD) { + doNestedLoopIntersect(pFinalRes, pRes); + } else { // for larger result, sort merge is employed + doSortIntersect(pFinalRes, pRes); + } +} + +static void queryResultUnion(tQueryResultset* pFinalRes, tQueryResultset* pRes) { + if (pRes->num == 0) { + tQueryResultClean(pRes); + return; + } + + int32_t total = pFinalRes->num + pRes->num; + void* tmp = realloc(pFinalRes->pRes, total * POINTER_BYTES); + if (tmp == NULL) { + return; + } + pFinalRes->pRes = tmp; + + memcpy(&pFinalRes->pRes[pFinalRes->num], pRes->pRes, POINTER_BYTES * pRes->num); + qsort(pFinalRes->pRes, total, POINTER_BYTES, tabObjPointerComparator); + + int32_t num = 1; + for (int32_t i = 1; i < total; ++i) { + if (pFinalRes->pRes[i] != pFinalRes->pRes[i - 1]) { + pFinalRes->pRes[num++] = pFinalRes->pRes[i]; + } + } + + if (num < total) { + memset(&pFinalRes->pRes[num], 0, POINTER_BYTES * (total - num)); + } + + pFinalRes->num = num; + + tQueryResultClean(pRes); +} + +static int32_t compareIntVal(const void* pLeft, const void* pRight) { + DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_INT64_VAL(pRight)); +} + +static int32_t compareIntDoubleVal(const void* pLeft, const void* pRight) { + DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_DOUBLE_VAL(pRight)); +} + +static int32_t compareDoubleVal(const void* pLeft, const void* pRight) { + DEFAULT_COMP(GET_DOUBLE_VAL(pLeft), GET_DOUBLE_VAL(pRight)); +} + +static int32_t compareDoubleIntVal(const void* pLeft, const void* pRight) { + double ret = (*(double*)pLeft) - (*(int64_t*)pRight); + if (fabs(ret) < DBL_EPSILON) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +static int32_t compareStrVal(const void* pLeft, const void* pRight) { + int32_t ret = strcmp(pLeft, pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +static int32_t compareWStrVal(const void* pLeft, const void* pRight) { + int32_t ret = wcscmp(pLeft, pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + const char* pattern = pRight; + const char* str = pLeft; + + int32_t ret = patternMatch(pattern, str, strlen(str), &pInfo); + + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; +} + +static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + const wchar_t* pattern = pRight; + const wchar_t* str = pLeft; + + int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo); + + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; +} + +static __compar_fn_t getFilterComparator(int32_t type, int32_t filterType, int32_t optr) { + __compar_fn_t comparator = NULL; + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_BOOL: { + if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { + comparator = compareIntVal; + } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { + comparator = compareIntDoubleVal; + } + break; + } + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { + comparator = compareDoubleIntVal; + } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { + comparator = compareDoubleVal; + } + break; + } + + case TSDB_DATA_TYPE_BINARY: { + assert(filterType == TSDB_DATA_TYPE_BINARY); + + if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + comparator = compareStrPatternComp; + } else { /* normal relational comparator */ + comparator = compareStrVal; + } + + break; + } + + case TSDB_DATA_TYPE_NCHAR: { + assert(filterType == TSDB_DATA_TYPE_NCHAR); + + if (optr == TSDB_RELATION_LIKE) { + comparator = compareWStrPatternComp; + } else { + comparator = compareWStrVal; + } + + break; + } + default: + comparator = compareIntVal; + break; + } + + return comparator; +} + +static void getTagColumnInfo(SSyntaxTreeFilterSupporter* pSupporter, SSchema* pSchema, int32_t* index, + int32_t* offset) { + *index = 0; + *offset = 0; + + // filter on table name(TBNAME) + if (strcasecmp(pSchema->name, TSQL_TBNAME_L) == 0) { + *index = TSDB_TBNAME_COLUMN_INDEX; + *offset = TSDB_TBNAME_COLUMN_INDEX; + return; + } + + while ((*index) < pSupporter->numOfTags) { + if (pSupporter->pTagSchema[*index].bytes == pSchema->bytes && + pSupporter->pTagSchema[*index].type == pSchema->type && + strcmp(pSupporter->pTagSchema[*index].name, pSchema->name) == 0) { + break; + } else { + (*offset) += pSupporter->pTagSchema[(*index)++].bytes; + } + } +} + +void filterPrepare(tSQLBinaryExpr* pExpr, void* param) { + if (pExpr->info != NULL) { + return; + } + + int32_t i = 0, offset = 0; + pExpr->info = calloc(1, sizeof(tQueryInfo)); + + tQueryInfo* pInfo = pExpr->info; + SSyntaxTreeFilterSupporter* pSupporter = (SSyntaxTreeFilterSupporter*)param; + + tVariant* pCond = pExpr->pRight->pVal; + SSchema* pSchema = pExpr->pLeft->pSchema; + + getTagColumnInfo(pSupporter, pSchema, &i, &offset); + assert((i >= 0 && i < TSDB_MAX_TAGS) || (i == TSDB_TBNAME_COLUMN_INDEX)); + assert((offset >= 0 && offset < TSDB_MAX_TAGS_LEN) || (offset == TSDB_TBNAME_COLUMN_INDEX)); + + pInfo->sch = *pSchema; + pInfo->colIdx = i; + pInfo->optr = pExpr->nSQLBinaryOptr; + pInfo->offset = offset; + pInfo->compare = getFilterComparator(pSchema->type, pCond->nType, pInfo->optr); + + tVariantAssign(&pInfo->q, pCond); + tVariantTypeSetType(&pInfo->q, pInfo->sch.type); +} + +void tSQLListTraverseDestroyInfo(void* param) { + if (param == NULL) { + return; + } + + tQueryInfo* pInfo = (tQueryInfo*)param; + tVariantDestroy(&(pInfo->q)); + free(param); +} + +static int32_t mgmtFilterMeterByIndex(STabObj* pMetric, tQueryResultset* pRes, char* pCond, int32_t condLen) { + SSchema* pTagSchema = (SSchema*)(pMetric->schema + pMetric->numOfColumns * sizeof(SSchema)); + + tSQLBinaryExpr* pExpr = NULL; + tSQLBinaryExprFromString(&pExpr, pTagSchema, pMetric->numOfTags, pCond, condLen); + + // failed to build expression, no result, return immediately + if (pExpr == NULL) { + mError("metric:%s, no result returned, error in super table query expression:%s", pMetric->meterId, pCond); + tfree(pCond); + + return TSDB_CODE_OPS_NOT_SUPPORT; + } else { // query according to the binary expression + SSyntaxTreeFilterSupporter s = {.pTagSchema = pTagSchema, .numOfTags = pMetric->numOfTags}; + SBinaryFilterSupp supp = {.fp = tSkipListNodeFilterCallback, .setupInfoFn = filterPrepare, .pExtInfo = &s}; + + tSQLBinaryExprTraverse(pExpr, pMetric->pSkipList, pRes, &supp); + tSQLBinaryExprDestroy(&pExpr, tSQLListTraverseDestroyInfo); + } + + tansformQueryResult(pRes); + + return TSDB_CODE_SUCCESS; +} + +int mgmtRetrieveMetersFromMetric(SMetricMetaMsg* pMsg, int32_t tableIndex, tQueryResultset* pRes) { + SMetricMetaElemMsg* pElem = (SMetricMetaElemMsg*)((char*)pMsg + pMsg->metaElem[tableIndex]); + STabObj* pMetric = mgmtGetMeter(pElem->meterId); + char* pCond = NULL; + char* tmpTableNameCond = NULL; + + // no table created in accordance with this metric. + if (pMetric->pSkipList == NULL || pMetric->pSkipList->nSize == 0) { + assert(pMetric->numOfMeters == 0); + return TSDB_CODE_SUCCESS; + } + + char* pQueryCond = (char*)pMsg + pElem->cond; + int32_t condLen = pElem->condLen; + + // transfer the unicode string to mbs binary expression + if (condLen > 0) { + pCond = calloc(1, (condLen + 1) * TSDB_NCHAR_SIZE); + + taosUcs4ToMbs(pQueryCond, condLen * TSDB_NCHAR_SIZE, pCond); + condLen = strlen(pCond) + 1; + mTrace("metric:%s len:%d, metric query condition:%s", pMetric->meterId, condLen, pCond); + } + + char* tablenameCond = (char*)pMsg + pElem->tableCond; + + if (pElem->tableCondLen > 0) { + tmpTableNameCond = calloc(1, pElem->tableCondLen + 1); + strncpy(tmpTableNameCond, tablenameCond, pElem->tableCondLen); + + mTrace("metric:%s rel:%d, len:%d, table name cond:%s", pMetric->meterId, pElem->rel, pElem->tableCondLen, + tmpTableNameCond); + } + + if (pElem->tableCondLen > 0 || condLen > 0) { + mgmtFilterByTableNameCond(pRes, tmpTableNameCond, pElem->tableCondLen, pMetric); + + bool noNextCal = (pRes->num == 0 && pElem->rel == TSDB_RELATION_AND); // no need to calculate next result + + if (!noNextCal && condLen > 0) { + tQueryResultset filterRes = {0}; + + int32_t ret = mgmtFilterMeterByIndex(pMetric, &filterRes, pCond, condLen); + if (ret != TSDB_CODE_SUCCESS) { + tfree(pCond); + tfree(tmpTableNameCond); + + return ret; + } + + // union or intersect of two results + assert(pElem->rel == TSDB_RELATION_AND || pElem->rel == TSDB_RELATION_OR); + + if (pElem->rel == TSDB_RELATION_AND) { + if (filterRes.num == 0 || pRes->num == 0) { // intersect two sets + tQueryResultClean(pRes); + } else { + queryResultIntersect(pRes, &filterRes); + } + } else { // union two sets + queryResultUnion(pRes, &filterRes); + } + + tQueryResultClean(&filterRes); + } + } else { + mTrace("metric:%s retrieve all meter, no query condition", pMetric->meterId); + pRes->num = tSkipListIterateList(pMetric->pSkipList, (tSkipListNode***)&pRes->pRes, NULL, NULL); + tansformQueryResult(pRes); + } + + tfree(pCond); + tfree(tmpTableNameCond); + + mTrace("metric:%s numOfRes:%d", pMetric->meterId, pRes->num); + return TSDB_CODE_SUCCESS; +} + +// todo refactor!!!!! +static char* getTagValueFromMeter(STabObj* pMeter, int32_t offset, void* param) { + if (offset == TSDB_TBNAME_COLUMN_INDEX) { + extractMeterName(pMeter->meterId, param); + return param; + } else { + char* tags = pMeter->pTagData + TSDB_METER_ID_LEN; // tag start position + return (tags + offset); + } +} + +bool tSkipListNodeFilterCallback(tSkipListNode* pNode, void* param) { + tQueryInfo* pInfo = (tQueryInfo*)param; + STabObj* pMeter = (STabObj*)pNode->pData; + + char name[TSDB_METER_NAME_LEN + 1] = {0}; + char* val = getTagValueFromMeter(pMeter, pInfo->offset, name); + int8_t type = pInfo->sch.type; + + int32_t ret = 0; + if (pInfo->q.nType == TSDB_DATA_TYPE_BINARY || pInfo->q.nType == TSDB_DATA_TYPE_NCHAR) { + ret = pInfo->compare(val, pInfo->q.pz); + } else { + tVariant t = {0}; + tVariantCreateFromBinary(&t, val, (uint32_t) pInfo->sch.bytes, type); + + ret = pInfo->compare(&t.i64Key, &pInfo->q.i64Key); + } + + switch (pInfo->optr) { + case TSDB_RELATION_EQUAL: { + return ret == 0; + } + case TSDB_RELATION_NOT_EQUAL: { + return ret != 0; + } + case TSDB_RELATION_LARGE_EQUAL: { + return ret >= 0; + } + case TSDB_RELATION_LARGE: { + return ret > 0; + } + case TSDB_RELATION_LESS_EQUAL: { + return ret <= 0; + } + case TSDB_RELATION_LESS: { + return ret < 0; + } + case TSDB_RELATION_LIKE: { + return ret == 0; + } + + default: + assert(false); + } + return true; +} diff --git a/src/system/detail/src/mgmtSystem.c b/src/system/detail/src/mgmtSystem.c new file mode 100644 index 0000000000..bb05c35e9b --- /dev/null +++ b/src/system/detail/src/mgmtSystem.c @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dnodeSystem.h" +#include "mgmt.h" +#include "tsdb.h" +#include "mgmtSystem.h" + +// global, not configurable +char mgmtDirectory[128]; +void * mgmtTmr; +void * mgmtQhandle = NULL; +void * mgmtTranQhandle = NULL; +void * mgmtStatisticTimer = NULL; +int mgmtShellConns = 0; +int mgmtDnodeConns = 0; +extern void * pShellConn; +extern void ** rpcQhandle; +extern SMgmtIpList mgmtIpList; +extern SMgmtIpList mgmtPublicIpList; +extern char mgmtIpStr[TSDB_MAX_MGMT_IPS][20]; +extern void * acctSdb; + +void mgmtCleanUpSystem() { + if (tsModuleStatus & (1 << TSDB_MOD_MGMT)) { + mTrace("mgmt is running, clean it up"); + taosTmrStopA(&mgmtStatisticTimer); + sdbCleanUpPeers(); + mgmtCleanupBalance(); + mgmtCleanUpDnodeInt(); + mgmtCleanUpShell(); + mgmtCleanUpMeters(); + mgmtCleanUpVgroups(); + mgmtCleanUpDbs(); + mgmtCleanUpDnodes(); + mgmtCleanUpUsers(); + mgmtCleanUpAccts(); + taosTmrCleanUp(mgmtTmr); + taosCleanUpScheduler(mgmtQhandle); + taosCleanUpScheduler(mgmtTranQhandle); + } else { + mgmtCleanUpRedirect(); + } + + mgmtTmr = NULL; + mgmtQhandle = NULL; + mgmtShellConns = 0; + mgmtDnodeConns = 0; + tclearModuleStatus(TSDB_MOD_MGMT); + pShellConn = NULL; + + mTrace("mgmt is cleaned up"); +} + +int mgmtStartSystem() { + mPrint("starting to initialize TDengine mgmt ..."); + + struct stat dirstat; + if (stat(mgmtDirectory, &dirstat) < 0) { + mkdir(mgmtDirectory, 0755); + } + + if (mgmtStartCheckMgmtRunning() != 0) { + mPrint("TDengine mgmt module already started..."); + return 0; + } + + int numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore / 2.0; + if (numOfThreads < 1) numOfThreads = 1; + mgmtQhandle = taosInitScheduler(tsMaxDnodes + tsMaxShellConns, numOfThreads, "mnode"); + + mgmtTranQhandle = taosInitScheduler(tsMaxDnodes + tsMaxShellConns, 1, "mnodeT"); + + mgmtTmr = taosTmrInit((tsMaxDnodes + tsMaxShellConns) * 3, 200, 3600000, "MND"); + if (mgmtTmr == NULL) { + mError("failed to init timer, exit"); + return -1; + } + + if (mgmtInitAccts() < 0) { + mError("failed to init accts"); + return -1; + } + + if (mgmtInitUsers() < 0) { + mError("failed to init users"); + return -1; + } + + if (mgmtInitDnodes() < 0) { + mError("failed to init dnodes"); + return -1; + } + + if (mgmtInitDbs() < 0) { + mError("failed to init dbs"); + return -1; + } + + if (mgmtInitVgroups() < 0) { + mError("failed to init vgroups"); + return -1; + } + + if (mgmtInitMeters() < 0) { + mError("failed to init meters"); + return -1; + } + + if (mgmtInitDnodeInt() < 0) { + mError("failed to init inter-mgmt communication"); + return -1; + } + + if (mgmtInitShell() < 0) { + mError("failed to init shell"); + return -1; + } + + if (sdbInitPeers(mgmtDirectory) < 0) { + mError("failed to init peers"); + return -1; + } + + if (mgmtInitBalance() < 0) { + mError("failed to init dnode balance") + } + + mgmtCheckAcct(); + + taosTmrReset(mgmtDoStatistic, tsStatusInterval * 30000, NULL, mgmtTmr, &mgmtStatisticTimer); + + mgmtStartMgmtTimer(); + + mPrint("TDengine mgmt is initialized successfully"); + + return 0; +} diff --git a/src/system/src/mgmtUser.c b/src/system/detail/src/mgmtUser.c similarity index 93% rename from src/system/src/mgmtUser.c rename to src/system/detail/src/mgmtUser.c index da690fffee..83e619841b 100644 --- a/src/system/src/mgmtUser.c +++ b/src/system/detail/src/mgmtUser.c @@ -13,10 +13,12 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include "mgmt.h" #include "tschemautil.h" +#include "ttime.h" void *userSdb = NULL; int tsUserUpdateSize; @@ -33,6 +35,11 @@ void *mgmtUserActionAfterBatchUpdate(void *row, char *str, int size, int *ssize) void *mgmtUserActionReset(void *row, char *str, int size, int *ssize); void *mgmtUserActionDestroy(void *row, char *str, int size, int *ssize); +SAcctObj *mgmtGetAcct(char *name); +void mgmtCreateRootAcct(); +int mgmtCheckUserLimit(SAcctObj *pAcct); +int mgmtCheckUserGrant(); + void mgmtUserActionInit() { mgmtUserActionFp[SDB_TYPE_INSERT] = mgmtUserActionInsert; mgmtUserActionFp[SDB_TYPE_DELETE] = mgmtUserActionDelete; @@ -67,6 +74,8 @@ int mgmtInitUsers() { return -1; } + mgmtCreateRootAcct(); + while (1) { pNode = sdbFetchRow(userSdb, pNode, (void **)&pUser); if (pUser == NULL) break; @@ -74,7 +83,7 @@ int mgmtInitUsers() { pUser->prev = NULL; pUser->next = NULL; - pAcct = &acctObj; + pAcct = mgmtGetAcct(pUser->acct); mgmtAddUserIntoAcct(pAcct, pUser); numOfUsers++; @@ -93,11 +102,11 @@ int mgmtUpdateUser(SUserObj *pUser) { return sdbUpdateRow(userSdb, pUser, 0, 1); int mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { SUserObj *pUser; + int code; - int numOfUsers = sdbGetNumOfRows(userSdb); - if (numOfUsers >= tsMaxUsers) { - mWarn("numOfUsers:%d, exceed tsMaxUsers:%d", numOfUsers, tsMaxUsers); - return TSDB_CODE_TOO_MANY_USERS; + code = mgmtCheckUserLimit(pAcct); + if (code != 0) { + return code; } pUser = (SUserObj *)sdbGetRow(userSdb, name); @@ -106,10 +115,15 @@ int mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { return TSDB_CODE_USER_ALREADY_EXIST; } + code = mgmtCheckUserGrant(); + if (code != 0) { + return code; + } + pUser = malloc(sizeof(SUserObj)); memset(pUser, 0, sizeof(SUserObj)); strcpy(pUser->user, name); - taosEncryptPass((uint8_t *)pass, strlen(pass), pUser->pass); + taosEncryptPass((uint8_t*) pass, strlen(pass), pUser->pass); strcpy(pUser->acct, pAcct->user); pUser->createdTime = taosGetTimestampMs(); pUser->superAuth = 0; @@ -118,7 +132,7 @@ int mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { pUser->superAuth = 1; } - int code = TSDB_CODE_SUCCESS; + code = TSDB_CODE_SUCCESS; if (sdbInsertRow(userSdb, pUser, 0) < 0) { tfree(pUser); code = TSDB_CODE_SDB_ERROR; @@ -224,14 +238,14 @@ int mgmtRetrieveUsers(SShowObj *pShow, char *data, int rows, SConnObj *pConn) { void *mgmtUserActionInsert(void *row, char *str, int size, int *ssize) { SUserObj *pUser = (SUserObj *)row; - SAcctObj *pAcct = &acctObj; + SAcctObj *pAcct = mgmtGetAcct(pUser->acct); mgmtAddUserIntoAcct(pAcct, pUser); return NULL; } void *mgmtUserActionDelete(void *row, char *str, int size, int *ssize) { SUserObj *pUser = (SUserObj *)row; - SAcctObj *pAcct = &acctObj; + SAcctObj *pAcct = mgmtGetAcct(pUser->acct); mgmtRemoveUserFromAcct(pAcct, pUser); return NULL; diff --git a/src/system/src/mgmtUtil.c b/src/system/detail/src/mgmtUtil.c similarity index 51% rename from src/system/src/mgmtUtil.c rename to src/system/detail/src/mgmtUtil.c index 318e95db2c..a427771e0f 100644 --- a/src/system/src/mgmtUtil.c +++ b/src/system/detail/src/mgmtUtil.c @@ -13,10 +13,18 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include +#include +#include #include "mgmt.h" +#include "mgmtUtil.h" +#include "textbuffer.h" +#include "tschemautil.h" +#include "tsqlfunction.h" +#include "vnodeTagMgmt.h" extern int cksumsize; @@ -42,3 +50,40 @@ bool mgmtMeterCreateFromMetric(STabObj* pMeterObj) { return pMeterObj->meterType bool mgmtIsMetric(STabObj* pMeterObj) { return pMeterObj->meterType == TSDB_METER_METRIC; } bool mgmtIsNormalMeter(STabObj* pMeterObj) { return !mgmtIsMetric(pMeterObj); } + +/** + * TODO: the tag offset value should be kept in memory to avoid dynamically calculating the value + * + * @param pMeter + * @param col + * @param pTagColSchema + * @return + */ +char* mgmtMeterGetTag(STabObj* pMeter, int32_t col, SSchema* pTagColSchema) { + if (!mgmtMeterCreateFromMetric(pMeter)) { + return NULL; + } + + STabObj* pMetric = mgmtGetMeter(pMeter->pTagData); + int32_t offset = mgmtGetTagsLength(pMetric, col) + TSDB_METER_ID_LEN; + assert(offset > 0); + + if (pTagColSchema != NULL) { + *pTagColSchema = ((SSchema*)pMetric->schema)[pMetric->numOfColumns + col]; + } + + return (pMeter->pTagData + offset); +} + +int32_t mgmtGetTagsLength(STabObj* pMetric, int32_t col) { // length before column col + assert(mgmtIsMetric(pMetric) && col >= 0); + + int32_t len = 0; + int32_t tagColumnIndexOffset = pMetric->numOfColumns; + + for (int32_t i = 0; i < pMetric->numOfTags && i < col; ++i) { + len += ((SSchema*)pMetric->schema)[tagColumnIndexOffset + i].bytes; + } + + return len; +} diff --git a/src/system/src/mgmtVgroup.c b/src/system/detail/src/mgmtVgroup.c similarity index 69% rename from src/system/src/mgmtVgroup.c rename to src/system/detail/src/mgmtVgroup.c index 1afcea17ad..5858be54ca 100644 --- a/src/system/src/mgmtVgroup.c +++ b/src/system/detail/src/mgmtVgroup.c @@ -13,11 +13,12 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include "mgmt.h" #include "tschemautil.h" -#pragma GCC diagnostic ignored "-Wunused-variable" +#include "tlog.h" void * vgSdb = NULL; int tsVgUpdateSize; @@ -37,6 +38,7 @@ void *mgmtVgroupActionBatchUpdate(void *row, char *str, int size, int *ssize); void *mgmtVgroupActionAfterBatchUpdate(void *row, char *str, int size, int *ssize); void *mgmtVgroupActionReset(void *row, char *str, int size, int *ssize); void *mgmtVgroupActionDestroy(void *row, char *str, int size, int *ssize); +bool mgmtCheckVnodeReady(SDnodeObj *pDnode, SVgObj *pVgroup, SVnodeGid *pVnode); void mgmtVgroupActionInit() { mgmtVgroupActionFp[SDB_TYPE_INSERT] = mgmtVgroupActionInsert; @@ -64,6 +66,9 @@ int mgmtInitVgroups() { mgmtVgroupActionInit(); + SVgObj tObj; + tsVgUpdateSize = tObj.updateEnd - (char *)&tObj; + vgSdb = sdbOpenTable(tsMaxVGroups, sizeof(SVgObj), "vgroups", SDB_KEYTYPE_AUTO, mgmtDirectory, mgmtVgroupAction); if (vgSdb == NULL) { mError("failed to init vgroup data"); @@ -81,17 +86,32 @@ int mgmtInitVgroups() { pVgroup->next = NULL; int size = sizeof(STabObj *) * pDb->cfg.maxSessions; pVgroup->meterList = (STabObj **)malloc(size); + if (pVgroup->meterList == NULL) { + mError("failed to malloc(size:%d) for the meterList of vgroups", size); + return -1; + } + memset(pVgroup->meterList, 0, size); pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); + if (pVgroup->idPool == NULL) { + mError("failed to taosInitIdPool for vgroups"); + free(pVgroup->meterList); + return -1; + } + taosIdPoolReinit(pVgroup->idPool); +#ifdef CLUSTER + if (pVgroup->vnodeGid[0].publicIp == 0) { + pVgroup->vnodeGid[0].publicIp = inet_addr(tsPublicIp); + pVgroup->vnodeGid[0].ip = inet_addr(tsPrivateIp); + sdbUpdateRow(vgSdb, pVgroup, tsVgUpdateSize, 1); + } +#endif - mgmtSetDnodeVgid(pVgroup->vnodeGid[0].vnode, pVgroup->vgId); + mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); } - SVgObj tObj; - tsVgUpdateSize = tObj.updateEnd - (char *)&tObj; - mTrace("vgroup is initialized"); return 0; } @@ -110,31 +130,6 @@ void mgmtProcessVgTimer(void *handle, void *tmrId) { pDb->vgTimer = NULL; } -bool mgmtAllocateVnode(SVgObj *pVgroup) { - int selectedVnode = -1; - SDnodeObj *pDnode = &dnodeObj; - - for (int i = 0; i < pDnode->numOfVnodes; i++) { - int vnode = (i + pDnode->lastAllocVnode) % pDnode->numOfVnodes; - if (pDnode->vload[vnode].vgId == 0 && pDnode->vload[vnode].status == TSDB_VN_STATUS_READY) { - selectedVnode = vnode; - break; - } - } - - if (selectedVnode == -1) { - mError("vgroup:%d alloc vnode failed, free vnodes:%d", pVgroup->vgId, pDnode->numOfFreeVnodes); - return false; - } else { - mTrace("vgroup:%d allocate vnode:%d, last allocated vnode:%d", pVgroup->vgId, selectedVnode, - pDnode->lastAllocVnode); - pVgroup->vnodeGid[0].vnode = selectedVnode; - pDnode->lastAllocVnode = selectedVnode + 1; - if (pDnode->lastAllocVnode >= pDnode->numOfVnodes) pDnode->lastAllocVnode = 0; - return true; - } -} - SVgObj *mgmtCreateVgroup(SDbObj *pDb) { SVgObj *pVgroup; int size; @@ -143,11 +138,12 @@ SVgObj *mgmtCreateVgroup(SDbObj *pDb) { pVgroup = (SVgObj *)malloc(size); memset(pVgroup, 0, size); strcpy(pVgroup->dbName, pDb->name); - pVgroup->numOfVnodes = 1; + pVgroup->numOfVnodes = pDb->cfg.replications; pVgroup->createdTime = taosGetTimestampMs(); - if (!mgmtAllocateVnode(pVgroup)) { - mWarn("no enough free dnode"); + // based on load balance, create a new one + if (mgmtAllocVnodes(pVgroup) != 0) { + mError("no enough free dnode"); free(pVgroup); pDb->vgStatus = TSDB_VG_STATUS_FULL; taosTmrReset(mgmtProcessVgTimer, 5000, pDb, mgmtTmr, &pDb->vgTimer); @@ -156,9 +152,11 @@ SVgObj *mgmtCreateVgroup(SDbObj *pDb) { sdbInsertRow(vgSdb, pVgroup, 0); - mTrace("vgroup:%d vnode:%d db:%s is created", pVgroup->vgId, pVgroup->vnodeGid[0].vnode, pDb->name); + mTrace("vgroup:%d, db:%s replica:%d is created", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); + for (int i = 0; i < pVgroup->numOfVnodes; ++i) + mTrace("dnode:%s, vgroup:%d, vnode:%d is created", taosIpStr(pVgroup->vnodeGid[i].ip), pVgroup->vgId, pVgroup->vnodeGid[i].vnode); - mgmtSendVPeersMsg(pVgroup, pDb); + mgmtSendVPeersMsg(pVgroup); return pVgroup; } @@ -175,7 +173,8 @@ int mgmtDropVgroup(SDbObj *pDb, SVgObj *pVgroup) { } } - mgmtSendFreeVnodeMsg(pVgroup->vnodeGid[0].vnode); + mTrace("vgroup:%d, db:%s replica:%d is deleted", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); + mgmtSendFreeVnodeMsg(pVgroup); sdbDeleteRow(vgSdb, pVgroup); return 0; @@ -223,12 +222,45 @@ int mgmtGetVgroupMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 2; - pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "vnode"); + pShow->bytes[cols] = 9; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "vgroup status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; + int maxReplica = 0; + SVgObj *pVgroup = pConn->pDb->pHead; + while (pVgroup != NULL) { + maxReplica = pVgroup->numOfVnodes > maxReplica ? pVgroup->numOfVnodes : maxReplica; + pVgroup = pVgroup->next; + } + + for (int i = 0; i < maxReplica; ++i) { + pShow->bytes[cols] = 16; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "ip"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "vnode"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 9; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "vnode status"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 16; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "public ip"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + } + pMeta->numOfColumns = htons(cols); pShow->numOfColumns = cols; @@ -249,7 +281,15 @@ int mgmtRetrieveVgroups(SShowObj *pShow, char *data, int rows, SConnObj *pConn) int cols = 0; char ipstr[20]; + int maxReplica = 0; + pVgroup = pConn->pDb->pHead; + while (pVgroup != NULL) { + maxReplica = pVgroup->numOfVnodes > maxReplica ? pVgroup->numOfVnodes : maxReplica; + pVgroup = pVgroup->next; + } + while (numOfRows < rows) { + // pShow->pNode = sdbFetchRow(vgSdb, pShow->pNode, (void **)&pVgroup); pVgroup = (SVgObj *)pShow->pNode; if (pVgroup == NULL) break; pShow->pNode = (void *)pVgroup->next; @@ -265,9 +305,34 @@ int mgmtRetrieveVgroups(SShowObj *pShow, char *data, int rows, SConnObj *pConn) cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pVgroup->vnodeGid[0].vnode; + strcpy(pWrite, pVgroup->lbState ? "updating" : "ready"); cols++; + for (int i = 0; i < maxReplica; ++i) { + tinet_ntoa(ipstr, pVgroup->vnodeGid[i].ip); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, ipstr); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pVgroup->vnodeGid[i].vnode; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + if (pVgroup->vnodeGid[i].ip != 0) { + bool ready = mgmtCheckVnodeReady(NULL, pVgroup, pVgroup->vnodeGid + i); + strcpy(pWrite, ready ? "ready" : "unsynced"); + } else { + strcpy(pWrite, "null"); + } + cols++; + + tinet_ntoa(ipstr, pVgroup->vnodeGid[i].publicIp); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, ipstr); + cols++; + } + numOfRows++; } @@ -287,7 +352,7 @@ void *mgmtVgroupActionInsert(void *row, char *str, int size, int *ssize) { pVgroup->numOfMeters = 0; pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); mgmtAddVgroupIntoDb(pDb, pVgroup); - mgmtSetDnodeVgid(pVgroup->vnodeGid[0].vnode, pVgroup->vgId); + mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); return NULL; } @@ -297,7 +362,7 @@ void *mgmtVgroupActionDelete(void *row, char *str, int size, int *ssize) { SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb != NULL) mgmtRemoveVgroupFromDb(pDb, pVgroup); - mgmtUnSetDnodeVgid(pVgroup->vnodeGid[0].vnode); + mgmtUnSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes); tfree(pVgroup->meterList); return NULL; @@ -306,6 +371,17 @@ void *mgmtVgroupActionDelete(void *row, char *str, int size, int *ssize) { void *mgmtVgroupActionUpdate(void *row, char *str, int size, int *ssize) { mgmtVgroupActionReset(row, str, size, ssize); SVgObj *pVgroup = (SVgObj *)row; + int oldTables = taosIdPoolMaxSize(pVgroup->idPool); + + SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + if (pDb != NULL) { + if (pDb->cfg.maxSessions != oldTables) { + mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions); + taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxSessions); + int size = sizeof(STabObj *) * pDb->cfg.maxSessions; + pVgroup->meterList = (STabObj **)realloc(pVgroup->meterList, size); + } + } mTrace("vgroup:%d update, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes); @@ -353,4 +429,4 @@ void *mgmtVgroupActionDestroy(void *row, char *str, int size, int *ssize) { if (pVgroup->meterList) tfree(pVgroup->meterList); tfree(row); return NULL; -} \ No newline at end of file +} diff --git a/src/system/src/vnodeCache.c b/src/system/detail/src/vnodeCache.c similarity index 83% rename from src/system/src/vnodeCache.c rename to src/system/detail/src/vnodeCache.c index 4c166136d0..8b51bc4609 100644 --- a/src/system/src/vnodeCache.c +++ b/src/system/detail/src/vnodeCache.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include #include @@ -68,7 +69,7 @@ void *vnodeOpenCachePool(int vnode) { int allocBlocks = MIN(pCfg->cacheNumOfBlocks.totalBlocks - blockId, maxAllocBlock); pMem = calloc(allocBlocks, pCfg->cacheBlockSize); if (pMem == NULL) { - dError("failed to allocate cache memory"); + dError("failed to allocate cache memory: %d", allocBlocks*pCfg->cacheBlockSize); goto _err_exit; } @@ -133,6 +134,7 @@ void *vnodeAllocateCacheInfo(SMeterObj *pObj) { pInfo->cacheBlocks = (SCacheBlock **)malloc(size); if (pInfo->cacheBlocks == NULL) { dError("id:%s, no memory for cacheBlocks", pObj->meterId); + tfree(pInfo); return NULL; } memset(pInfo->cacheBlocks, 0, size); @@ -623,7 +625,7 @@ int vnodeQueryFromCache(SMeterObj *pObj, SQuery *pQuery) { } else { // check each data one by one // set the input column data for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { - int16_t colIdx = pQuery->pFilterInfo[k].pFilter.colIdx; + int16_t colIdx = pQuery->pFilterInfo[k].info.colIdx; if (colIdx < 0) { // current data has not specified column pQuery->pFilterInfo[k].pData = NULL; @@ -910,7 +912,7 @@ void vnodeSetCommitQuery(SMeterObj *pObj, SQuery *pQuery) { pColIndexEx->colId = pObj->schema[col].colId; pColIndexEx->colIdx = col; pColIndexEx->colIdxInBuf = col; - pColIndexEx->isTag = false; + pColIndexEx->flag = TSDB_COL_NORMAL; } pQuery->slot = pInfo->commitSlot; @@ -956,6 +958,181 @@ void vnodeSetCommitQuery(SMeterObj *pObj, SQuery *pQuery) { } } +int vnodeSyncRetrieveVnodeStatistic(int vnode, int fd) { + SVnodeObj *pVnode = vnodeList + vnode; + if (taosWriteMsg(fd, &(pVnode->vnodeStatistic.pointsWritten), sizeof(int64_t)) < 0) return -1; + if (taosWriteMsg(fd, &(pVnode->vnodeStatistic.totalStorage), sizeof(int64_t)) < 0) return -1; + if (taosWriteMsg(fd, &(pVnode->vnodeStatistic.compStorage), sizeof(int64_t)) < 0) return -1; + + return 0; +} + +int vnodeSyncRestoreVnodeStatistic(int vnode, int fd) { + SVnodeObj *pVnode = vnodeList + vnode; + if (taosReadMsg(fd, &(pVnode->vnodeStatistic.pointsWritten), sizeof(int64_t)) < 0) return -1; + if (taosReadMsg(fd, &(pVnode->vnodeStatistic.totalStorage), sizeof(int64_t)) < 0) return -1; + if (taosReadMsg(fd, &(pVnode->vnodeStatistic.compStorage), sizeof(int64_t)) < 0) return -1; + + return 0; +} + +int vnodeSyncRetrieveCache(int vnode, int fd) { + int32_t sid, slot, points; + SVnodeObj * pVnode; + SMeterObj * pObj; + SCacheInfo * pInfo; + SCacheBlock *pBlock; + int blocksSent, pointsSent; + + pVnode = vnodeList + vnode; + points = 0; + SVnodeCfg *pCfg = &vnodeList[vnode].cfg; + + for (sid = 0; sid < pCfg->maxSessions; ++sid) { + pObj = pVnode->meterList[sid]; + if (pObj == NULL) continue; + + pInfo = (SCacheInfo *)pObj->pCache; + if (pInfo == NULL) continue; + + // write sid first + if (taosWriteMsg(fd, &sid, sizeof(sid)) <= 0) return -1; + if (taosWriteMsg(fd, &(pObj->lastKey), sizeof(pObj->lastKey)) <= 0) return -1; + if (taosWriteMsg(fd, &(pObj->lastKeyOnFile), sizeof(pObj->lastKeyOnFile)) <= 0) return -1; + if (taosWriteMsg(fd, &(pInfo->commitPoint), sizeof(pInfo->commitPoint)) <= 0) return -1; + + dTrace("vid:%d sid:%d id:%s, send lastKey:%lld lastKeyOnFile:%lld", vnode, sid, pObj->meterId, pObj->lastKey, + pObj->lastKeyOnFile); + + slot = pInfo->commitSlot; + blocksSent = 0; + pointsSent = 0; + + while (pInfo->numOfBlocks > 0) { + pBlock = pInfo->cacheBlocks[slot]; + if (pBlock->numOfPoints == 0) break; + + // write the number of points + points = pBlock->numOfPoints; + if (taosWriteMsg(fd, &(points), sizeof(points)) <= 0) return -1; + + // write the data + for (int col = 0; col < pObj->numOfColumns; ++col) + if (taosWriteMsg(fd, pBlock->offset[col], pObj->schema[col].bytes * points) <= 0) return -1; + + TSKEY lastKey = *((TSKEY *)(pBlock->offset[0] + pObj->schema[0].bytes * (points - 1))); + dTrace("vid:%d sid:%d id:%s, cache block is sent, points:%d lastKey:%ld", vnode, sid, pObj->meterId, points, + lastKey); + + blocksSent++; + pointsSent += pBlock->numOfPoints; + if (slot == pInfo->currentSlot) break; + + slot = (slot + 1) % pInfo->maxBlocks; + } + + // set number of points as zero at the end + points = 0; + if (taosWriteMsg(fd, &(points), sizeof(points)) <= 0) return -1; + } + + sid = -1; + if (taosWriteMsg(fd, &sid, sizeof(sid)) < 0) return -1; + if (vnodeSyncRetrieveVnodeStatistic(vnode, fd) < 0) return -1; + + return 0; +} + +int vnodeSyncRestoreCache(int vnode, int fd) { + int32_t sid, points, i, slot; + SMeterObj * pObj; + SCacheInfo * pInfo; + SCacheBlock *pBlock; + int blocksReceived, pointsReceived; + int numOfBlocks; + SVnodeCfg * pCfg = &vnodeList[vnode].cfg; + SCachePool * pPool = (SCachePool *)vnodeList[vnode].pCachePool; + + while (1) { + // read sid first + + if (taosReadMsg(fd, &sid, sizeof(sid)) <= 0) return -1; + if (sid >= pCfg->maxSessions) { + dError("vid:%d, restore cache, sid:%d is messed up", vnode, sid); + return -1; + } + if (sid < 0) break; + + pObj = vnodeList[vnode].meterList[sid]; + if (pObj == NULL) { + dError("vid:%d sid:%d, meter is not there", vnode, sid); + vnodeSendMeterCfgMsg(vnode, sid); + return -1; + } + + pInfo = (SCacheInfo *)pObj->pCache; + numOfBlocks = pInfo->numOfBlocks; + pthread_mutex_lock(&pPool->vmutex); + for (i = 0; i < numOfBlocks; ++i) { + slot = (pInfo->currentSlot - i + pInfo->maxBlocks) % pInfo->maxBlocks; + pBlock = pInfo->cacheBlocks[slot]; + vnodeFreeCacheBlock(pBlock); + } + pthread_mutex_unlock(&pPool->vmutex); + + pInfo->unCommittedBlocks = 0; + if (taosReadMsg(fd, &(pObj->lastKey), sizeof(pObj->lastKey)) <= 0) return -1; + if (taosReadMsg(fd, &(pObj->lastKeyOnFile), sizeof(pObj->lastKeyOnFile)) <= 0) return -1; + if (taosReadMsg(fd, &(pInfo->commitPoint), sizeof(pInfo->commitPoint)) <= 0) return -1; + + dTrace("vid:%d sid:%d id:%s, commitPoint:%d lastKeyOnFile:%ld", vnode, sid, pObj->meterId, pInfo->commitPoint, + pObj->lastKeyOnFile); + + if (vnodeList[pObj->vnode].lastKey < pObj->lastKey) vnodeList[pObj->vnode].lastKey = pObj->lastKey; + + if (vnodeList[pObj->vnode].lastKeyOnFile < pObj->lastKeyOnFile) + vnodeList[pObj->vnode].lastKeyOnFile = pObj->lastKeyOnFile; + + pInfo->currentSlot = -1; + pInfo->commitSlot = 0; + memset(pInfo->cacheBlocks, 0, sizeof(SCacheBlock *) * pInfo->maxBlocks); + blocksReceived = 0; + pointsReceived = 0; + pObj->freePoints = pObj->pointsPerBlock * pInfo->maxBlocks; + + while (1) { + // read number of points; + points = 0; + if (taosReadMsg(fd, &points, sizeof(points)) <= 0) return -1; + if (points == 0) break; + + if (vnodeAllocateCacheBlock(pObj) < 0) return -1; + pBlock = pInfo->cacheBlocks[pInfo->currentSlot]; + pBlock->numOfPoints = points; + + // read the data + for (int col = 0; col < pObj->numOfColumns; ++col) + if (taosReadMsg(fd, pBlock->offset[col], pObj->schema[col].bytes * points) <= 0) return -1; + + __sync_fetch_and_sub(&pObj->freePoints, points); + blocksReceived++; + pointsReceived += points; + pObj->lastKey = *((TSKEY *)(pBlock->offset[0] + pObj->schema[0].bytes * (points - 1))); + if (vnodeList[pObj->vnode].lastKey < pObj->lastKey) vnodeList[pObj->vnode].lastKey = pObj->lastKey; + + if (vnodeList[pObj->vnode].firstKey > *(TSKEY *)(pBlock->offset[0])) + vnodeList[pObj->vnode].firstKey = *(TSKEY *)(pBlock->offset[0]); + + dTrace("vid:%d sid:%d id:%s, cache block is received, points:%d lastKey:%ld", vnode, sid, pObj->meterId, points, + pObj->lastKey); + } + } + + if (vnodeSyncRestoreVnodeStatistic(pObj->vnode, fd) < 0) return -1; + + return 0; +} + int vnodeIsCacheCommitted(SMeterObj *pObj) { if (pObj->pCache == NULL) return 1; diff --git a/src/system/src/vnodeCommit.c b/src/system/detail/src/vnodeCommit.c similarity index 98% rename from src/system/src/vnodeCommit.c rename to src/system/detail/src/vnodeCommit.c index a14e8118c4..af14f2be4a 100644 --- a/src/system/src/vnodeCommit.c +++ b/src/system/detail/src/vnodeCommit.c @@ -48,7 +48,7 @@ int vnodeOpenCommitLog(int vnode, uint64_t firstV) { dTrace("vid:%d, logfd:%d, open file:%s success", vnode, pVnode->logFd, fileName); if (posix_fallocate64(pVnode->logFd, 0, pVnode->mappingSize) != 0) { - dError("vid:%d, logfd:%d, failed to alloc file size:%d reason:%s", vnode, pVnode->logFd, pVnode->mappingSize, strerror(errno)); + dError("vid:%d, logfd:%d, failed to alloc file size:%d, reason:%s", vnode, pVnode->logFd, pVnode->mappingSize, strerror(errno)); perror("fallocate failed"); goto _err_log_open; } @@ -143,8 +143,8 @@ size_t vnodeRestoreDataFromLog(int vnode, char *fileName, uint64_t *firstV) { dError("vid:%d, out of memory", vnode); goto _error; } - TSKEY now = taosGetTimestamp(pVnode->cfg.precision); + SCommitHead head; int simpleCheck = 0; while (1) { diff --git a/src/system/src/vnodeFile.c b/src/system/detail/src/vnodeFile.c similarity index 91% rename from src/system/src/vnodeFile.c rename to src/system/detail/src/vnodeFile.c index 5b08ecd39b..ce133b2f89 100644 --- a/src/system/src/vnodeFile.c +++ b/src/system/detail/src/vnodeFile.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include #include @@ -65,6 +66,12 @@ int vnodeRecoverHeadFile(int vnode, int fileId); int vnodeRecoverDataFile(int vnode, int fileId); int vnodeForwardStartPosition(SQuery *pQuery, SCompBlock *pBlock, int32_t slotIdx, SVnodeObj *pVnode, SMeterObj *pObj); int vnodeCheckNewHeaderFile(int fd, SVnodeObj *pVnode); +char* vnodeGetDataDir(int vnode, int fileId); +char* vnodeGetDiskFromHeadFile(char *headName); +void vnodeAdustVnodeFile(SVnodeObj *pVnode); +int vnodeSyncRetrieveFile(int vnode, int fd, uint32_t peerFid, uint64_t *fmagic); +int vnodeSyncRestoreFile(int vnode, int sfd); +void vnodeAdjustFileTier(int vnode); void vnodeGetHeadDataLname(char *headName, char *dataName, char *lastName, int vnode, int fileId) { if (headName != NULL) sprintf(headName, "%s/vnode%d/db/v%df%d.head", tsDirectory, vnode, vnode, fileId); @@ -113,16 +120,23 @@ int vnodeCreateHeadDataFile(int vnode, int fileId, char *headName, char *dataNam char dDataName[TSDB_FILENAME_LEN]; char dLastName[TSDB_FILENAME_LEN]; - vnodeCreateDataDirIfNeeded(vnode, dataDir); + char *path = vnodeGetDataDir(vnode, fileId); + if (path == NULL) { + return -1; + } + + vnodeCreateDataDirIfNeeded(vnode, path); vnodeGetHeadDataLname(headName, dataName, lastName, vnode, fileId); - vnodeGetHeadDataDname(dHeadName, dDataName, dLastName, vnode, fileId, dataDir); + vnodeGetHeadDataDname(dHeadName, dDataName, dLastName, vnode, fileId, path); if (symlink(dHeadName, headName) != 0) return -1; if (symlink(dDataName, dataName) != 0) return -1; if (symlink(dLastName, lastName) != 0) return -1; - dTrace("vid:%d, fileId:%d, empty header file:%s dataFile:%s lastFile:%s on disk:%s is created ", - vnode, fileId, headName, dataName, lastName, tsDirectory); + dTrace( + "vid:%d, fileId:%d, empty header file:%s dataFile:%s lastFile:%s on " + "disk:%s is created ", + vnode, fileId, headName, dataName, lastName, path); return 0; } @@ -188,7 +202,7 @@ int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast) { if (pVnode->lastKeyOnFile == 0) { if (pCfg->daysPerFile == 0) pCfg->daysPerFile = 10; pVnode->fileId = pVnode->firstKey / tsMsPerDay[pVnode->cfg.precision] / pCfg->daysPerFile; - pVnode->lastKeyOnFile = (long)(pVnode->fileId + 1) * pCfg->daysPerFile * tsMsPerDay[pVnode->cfg.precision] - 1; + pVnode->lastKeyOnFile = (int64_t)(pVnode->fileId + 1) * pCfg->daysPerFile * tsMsPerDay[pVnode->cfg.precision] - 1; pVnode->numOfFiles = 1; vnodeCreateEmptyCompFile(vnode, pVnode->fileId); } @@ -204,21 +218,31 @@ int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast) { filesAdded = numOfFiles - pVnode->numOfFiles + 1; for (int i = 0; i < filesAdded; ++i) { fileId = pVnode->fileId - pVnode->numOfFiles - i; - if (vnodeCreateEmptyCompFile(vnode, fileId) < 0) return -1; + if (vnodeCreateEmptyCompFile(vnode, fileId) < 0) +#ifdef CLUSTER + return vnodeRecoverFromPeer(pVnode, fileId); +#else + return -1; +#endif } } else if (numOfFiles < 0) { // create empty header files forward pVnode->fileId++; - if (vnodeCreateEmptyCompFile(vnode, pVnode->fileId) < 0) return -1; - pVnode->lastKeyOnFile += (long)tsMsPerDay[pVnode->cfg.precision] * pCfg->daysPerFile; + if (vnodeCreateEmptyCompFile(vnode, pVnode->fileId) < 0) +#ifdef CLUSTER + return vnodeRecoverFromPeer(pVnode, pVnode->fileId); +#else + return -1; +#endif + pVnode->lastKeyOnFile += (int64_t)tsMsPerDay[pVnode->cfg.precision] * pCfg->daysPerFile; filesAdded = 1; numOfFiles = 0; // hacker way } fileId = pVnode->fileId - numOfFiles; pVnode->commitLastKey = - pVnode->lastKeyOnFile - (long)numOfFiles * tsMsPerDay[pVnode->cfg.precision] * pCfg->daysPerFile; - pVnode->commitFirstKey = pVnode->commitLastKey - (long)tsMsPerDay[pVnode->cfg.precision] * pCfg->daysPerFile + 1; + pVnode->lastKeyOnFile - (int64_t)numOfFiles * tsMsPerDay[pVnode->cfg.precision] * pCfg->daysPerFile; + pVnode->commitFirstKey = pVnode->commitLastKey - (int64_t)tsMsPerDay[pVnode->cfg.precision] * pCfg->daysPerFile + 1; pVnode->commitFileId = fileId; pVnode->numOfFiles = pVnode->numOfFiles + filesAdded; @@ -253,6 +277,7 @@ int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast) { if (pVnode->hfd < 0) { dError("vid:%d, failed to open head file:%s, reason:%s", vnode, pVnode->cfn, strerror(errno)); taosLogError("vid:%d, failed to open head file:%s, reason:%s", vnode, pVnode->cfn, strerror(errno)); + vnodeRecoverFromPeer(pVnode, fileId); goto _error; } @@ -261,6 +286,7 @@ int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast) { if (filestat.st_size < minSize) { dError("vid:%d, head file:%s corrupted", vnode, pVnode->cfn); taosLogError("vid:%d, head file:%s corrupted", vnode, pVnode->cfn); + vnodeRecoverFromPeer(pVnode, fileId); goto _error; } @@ -278,6 +304,7 @@ int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast) { if (pVnode->dfd < 0) { dError("vid:%d, failed to open data file:%s, reason:%s", vnode, name, strerror(errno)); taosLogError("vid:%d, failed to open data file:%s, reason:%s", vnode, name, strerror(errno)); + vnodeRecoverFromPeer(pVnode, fileId); goto _error; } @@ -286,6 +313,7 @@ int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast) { if (filestat.st_size < TSDB_FILE_HEADER_LEN) { dError("vid:%d, data file:%s corrupted", vnode, name); taosLogError("vid:%d, data file:%s corrupted", vnode, name); + vnodeRecoverFromPeer(pVnode, fileId); goto _error; } else { dTrace("vid:%d, data file:%s is opened to write", vnode, name); @@ -296,6 +324,7 @@ int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast) { if (pVnode->lfd < 0) { dError("vid:%d, failed to open last file:%s, reason:%s", vnode, pVnode->lfn, strerror(errno)); taosLogError("vid:%d, failed to open last file:%s, reason:%s", vnode, pVnode->lfn, strerror(errno)); + vnodeRecoverFromPeer(pVnode, fileId); goto _error; } @@ -304,6 +333,7 @@ int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast) { if (filestat.st_size < TSDB_FILE_HEADER_LEN) { dError("vid:%d, last file:%s corrupted", vnode, pVnode->lfn); taosLogError("vid:%d, last file:%s corrupted", vnode, pVnode->lfn); + vnodeRecoverFromPeer(pVnode, fileId); goto _error; } @@ -323,7 +353,14 @@ int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast) { int size = sizeof(SCompHeader) * pVnode->cfg.maxSessions + sizeof(TSCKSUM); char *temp = malloc(size); + if (NULL == temp) { + dError("vid:%d, malloc failed", vnode); + taosLogError("vid:%d, malloc failed", vnode); + //vnodeRecoverFromPeer(pVnode, fileId); + goto _error; + } memset(temp, 0, size); + taosCalcChecksumAppend(0, (uint8_t *)temp, size); twrite(pVnode->nfd, temp, size); free(temp); @@ -364,6 +401,10 @@ void vnodeRemoveFile(int vnode, int fileId) { pVnode = vnodeList + vnode; vnodeGetHeadDataLname(headName, dataName, lastName, vnode, fileId); + char *path = vnodeGetDiskFromHeadFile(headName); + if (path == NULL) { + return ; + } vnodeGetDnameFromLname(headName, dataName, lastName, dHeadName, dDataName, dLastName); int fd = open(headName, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); @@ -380,13 +421,12 @@ void vnodeRemoveFile(int vnode, int fileId) { remove(dDataName); remove(dLastName); - dTrace("vid:%d fileId:%d on disk: %s is removed, numOfFiles:%d maxFiles:%d", vnode, fileId, tsDirectory, + dTrace("vid:%d fileId:%d on disk: %s is removed, numOfFiles:%d maxFiles:%d", vnode, fileId, path, pVnode->numOfFiles, pVnode->maxFiles); } void vnodeCloseCommitFiles(SVnodeObj *pVnode) { char dpath[TSDB_FILENAME_LEN] = "\0"; - int fileId; int ret; // Check new if new header file is correct @@ -432,16 +472,7 @@ void vnodeCloseCommitFiles(SVnodeObj *pVnode) { pVnode->tfd = 0; dTrace("vid:%d, %s and %s is saved", pVnode->vnode, pVnode->cfn, pVnode->lfn); - - // Retention policy here - fileId = pVnode->fileId - pVnode->numOfFiles + 1; - int cfile = taosGetTimestamp(pVnode->cfg.precision)/pVnode->cfg.daysPerFile/tsMsPerDay[pVnode->cfg.precision]; - while (fileId <= cfile - pVnode->maxFiles) { - vnodeRemoveFile(pVnode->vnode, fileId); - pVnode->numOfFiles--; - fileId++; - } - + vnodeAdustVnodeFile(pVnode); vnodeSaveAllMeterObjToFile(pVnode->vnode); return; @@ -458,12 +489,12 @@ void *vnodeCommitMultiToFile(SVnodeObj *pVnode, int ssid, int esid) { SCompHeader * pHeader; SMeterInfo * meterInfo = NULL, *pMeter = NULL; SQuery query; - SColumnFilter colList[TSDB_MAX_COLUMNS] = {0}; + SColumnInfoEx colList[TSDB_MAX_COLUMNS] = {0}; SSqlFunctionExpr pExprs[TSDB_MAX_COLUMNS] = {0}; int commitAgain; int headLen, sid, col; - long pointsRead; - long pointsReadLast; + int64_t pointsRead; + int64_t pointsReadLast; SCompBlock * pCompBlock = NULL; SVnodeCfg * pCfg = &pVnode->cfg; TSCKSUM chksum; @@ -474,6 +505,7 @@ void *vnodeCommitMultiToFile(SVnodeObj *pVnode, int ssid, int esid) { pVnode->lastKey, ssid, esid); if (pVnode->lastKey == 0) goto _over; + vnodeCloseAllSyncFds(vnode); vnodeRenewCommitLog(vnode); // get the MAX consumption buffer for this vnode @@ -549,12 +581,15 @@ _again: if (read(pVnode->hfd, tmem, tmsize) <= 0) { dError("vid:%d, failed to read old header file:%s", vnode, pVnode->cfn); taosLogError("vid:%d, failed to read old header file:%s", vnode, pVnode->cfn); + vnodeRecoverFromPeer(pVnode, pVnode->commitFileId); goto _over; } else { if (!taosCheckChecksumWhole((uint8_t *)tmem, tmsize)) { dError("vid:%d, failed to read old header file:%s since comp header offset is broken", vnode, pVnode->cfn); taosLogError("vid:%d, failed to read old header file:%s since comp header offset is broken", - vnode, pVnode->cfn); + vnode, pVnode->cfn); + + vnodeRecoverFromPeer(pVnode, pVnode->commitFileId); goto _over; } } @@ -586,9 +621,10 @@ _again: if (read(pVnode->hfd, &compInfo, sizeof(compInfo)) == sizeof(compInfo)) { if (!taosCheckChecksumWhole((uint8_t *)(&compInfo), sizeof(SCompInfo))) { dError("vid:%d sid:%d id:%s, failed to read compinfo in file:%s since checksum mismatch", - vnode, sid, pObj->meterId, pVnode->cfn); + vnode, sid, pObj->meterId, pVnode->cfn); taosLogError("vid:%d sid:%d id:%s, failed to read compinfo in file:%s since checksum mismatch", - vnode, sid, pObj->meterId, pVnode->cfn); + vnode, sid, pObj->meterId, pVnode->cfn); + vnodeRecoverFromPeer(pVnode, pVnode->commitFileId); goto _over; } else { if (pObj->uid == compInfo.uid) { @@ -602,12 +638,13 @@ _again: } } else { dTrace("vid:%d sid:%d id:%s, uid:%ld is not matched w/ old:%ld, old data will be thrown away", - vnode, sid, pObj->meterId, pObj->uid, compInfo.uid); + vnode, sid, pObj->meterId, pObj->uid, compInfo.uid); pMeter->oldNumOfBlocks = 0; } } } else { dError("vid:%d sid:%d id:%s, failed to read compinfo in file:%s", vnode, sid, pObj->meterId, pVnode->cfn); + vnodeRecoverFromPeer(pVnode, pVnode->commitFileId); goto _over; } } @@ -664,16 +701,15 @@ _again: pVnode->dfSize = pCompBlock->offset + pMeter->lastBlock.len; } else { if (ssid == 0) { - // Here, pVnode->tfd != -1 - assert(pVnode->tfd != -1); - pCompBlock->offset = lseek(pVnode->tfd, 0, SEEK_END); + assert(pCompBlock->last && pVnode->tfd != -1); + pCompBlock->offset = lseek(pVnode->tfd, 0, SEEK_END); lseek(pVnode->lfd, pMeter->lastBlock.offset, SEEK_SET); tsendfile(pVnode->tfd, pVnode->lfd, NULL, pMeter->lastBlock.len); pVnode->lfSize = pCompBlock->offset + pMeter->lastBlock.len; } else { - // Here, pVnode->tfd = -1 assert(pVnode->tfd == -1); } + } headLen += sizeof(SCompBlock); @@ -719,9 +755,10 @@ _again: pMeter->newNumOfBlocks++; pMeter->committedPoints += (pointsRead - pointsReadLast); - dTrace("vid:%d sid:%d id:%s, pointsRead:%d, pointsReadLast:%d lastKey:%lld, slot:%d pos:%d newNumOfBlocks:%d headLen:%d", - pObj->vnode, pObj->sid, pObj->meterId, pointsRead, pointsReadLast, pObj->lastKeyOnFile, query.slot, - query.pos, pMeter->newNumOfBlocks, headLen); + dTrace("vid:%d sid:%d id:%s, pointsRead:%d, pointsReadLast:%d lastKey:%lld, " + "slot:%d pos:%d newNumOfBlocks:%d headLen:%d", + pObj->vnode, pObj->sid, pObj->meterId, pointsRead, pointsReadLast, pObj->lastKeyOnFile, query.slot, query.pos, + pMeter->newNumOfBlocks, headLen); if (pointsRead < pObj->pointsPerFileBlock || query.keyIsMet) break; @@ -785,6 +822,7 @@ _again: if (twrite(pVnode->nfd, tmem, tmsize) <= 0) { dError("vid:%d sid:%d id:%s, failed to write:%s, error:%s", vnode, sid, pObj->meterId, pVnode->nfn, strerror(errno)); + vnodeRecoverFromPeer(pVnode, pVnode->commitFileId); goto _over; } @@ -808,6 +846,7 @@ _again: if (twrite(pVnode->nfd, &compInfo, sizeof(compInfo)) <= 0) { dError("vid:%d sid:%d id:%s, failed to write:%s, reason:%s", vnode, sid, pObj->meterId, pVnode->nfn, strerror(errno)); + vnodeRecoverFromPeer(pVnode, pVnode->commitFileId); goto _over; } @@ -832,6 +871,7 @@ _again: if (twrite(pVnode->nfd, hmem + pMeter->tempHeadOffset, pMeter->newNumOfBlocks * sizeof(SCompBlock)) <= 0) { dError("vid:%d sid:%d id:%s, failed to write:%s, reason:%s", vnode, sid, pObj->meterId, pVnode->nfn, strerror(errno)); + vnodeRecoverFromPeer(pVnode, pVnode->commitFileId); goto _over; } } @@ -868,6 +908,7 @@ _over: tfree(buffer); tfree(pOldCompBlocks); + vnodeBroadcastStatusToUnsyncedPeer(pVnode); dPrint("vid:%d, committing is over", vnode); return pVnode; @@ -904,9 +945,9 @@ int vnodeGetCompBlockInfo(SMeterObj *pObj, SQuery *pQuery) { pthread_mutex_unlock(&(pVnode->vmutex)); if (pQuery->hfd < 0) { - dError("vid:%d sid:%d id:%s, failed to open head file:%s, reason:%s", - pObj->vnode, pObj->sid, pObj->meterId, fileName, strerror(errno)); - return -TSDB_CODE_FILE_CORRUPTED; + dError("vid:%d sid:%d id:%s, failed to open head file:%s, reason:%s", pObj->vnode, pObj->sid, pObj->meterId, + fileName, strerror(errno)); + return vnodeRecoverFromPeer(pVnode, pQuery->fileId); } int tmsize = sizeof(SCompHeader) * pCfg->maxSessions + sizeof(TSCKSUM); @@ -923,7 +964,7 @@ int vnodeGetCompBlockInfo(SMeterObj *pObj, SQuery *pQuery) { taosLogError("vid:%d sid:%d id:%s, file:%s failed to read comp header", pObj->vnode, pObj->sid, pObj->meterId, fileName); tfree(buffer); - return -TSDB_CODE_FILE_CORRUPTED; + return vnodeRecoverFromPeer(pVnode, pQuery->fileId); } if (!taosCheckChecksumWhole((uint8_t *)buffer, tmsize)) { @@ -932,7 +973,7 @@ int vnodeGetCompBlockInfo(SMeterObj *pObj, SQuery *pQuery) { taosLogError("vid:%d sid:%d id:%s, file:%s comp header offset is broken", pObj->vnode, pObj->sid, pObj->meterId, fileName); tfree(buffer); - return -TSDB_CODE_FILE_CORRUPTED; + return vnodeRecoverFromPeer(pVnode, pQuery->fileId); } compHeader = ((SCompHeader *)buffer)[pObj->sid]; tfree(buffer); @@ -941,11 +982,10 @@ int vnodeGetCompBlockInfo(SMeterObj *pObj, SQuery *pQuery) { lseek(pQuery->hfd, compHeader.compInfoOffset, SEEK_SET); read(pQuery->hfd, &compInfo, sizeof(SCompInfo)); if (!taosCheckChecksumWhole((uint8_t *)(&compInfo), sizeof(SCompInfo))) { - dError("vid:%d sid:%d id:%s, file:%s compInfo checksum mismatch", - pObj->vnode, pObj->sid, pObj->meterId, fileName); - taosLogError("vid:%d sid:%d id:%s, file:%s compInfo checksum mismatch", - pObj->vnode, pObj->sid, pObj->meterId, fileName); - return -TSDB_CODE_FILE_CORRUPTED; + dError("vid:%d sid:%d id:%s, file:%s compInfo checksum mismatch", pObj->vnode, pObj->sid, pObj->meterId, fileName); + taosLogError("vid:%d sid:%d id:%s, file:%s compInfo checksum mismatch", pObj->vnode, pObj->sid, pObj->meterId, + fileName); + return vnodeRecoverFromPeer(pVnode, pQuery->fileId); } if (compInfo.numOfBlocks <= 0) return 0; if (compInfo.uid != pObj->uid) return 0; @@ -954,16 +994,16 @@ int vnodeGetCompBlockInfo(SMeterObj *pObj, SQuery *pQuery) { pQuery->pBlock = (SCompBlock *)calloc(1, (sizeof(SCompBlock) + sizeof(SField *)) * compInfo.numOfBlocks); pQuery->pFields = (SField **)((char *)pQuery->pBlock + sizeof(SCompBlock) * compInfo.numOfBlocks); - /* char *pBlock = (char *)pQuery->pBlockFields + sizeof(SCompBlockFields)*compInfo.numOfBlocks; */ + /* char *pBlock = (char *)pQuery->pBlockFields + + * sizeof(SCompBlockFields)*compInfo.numOfBlocks; */ read(pQuery->hfd, pQuery->pBlock, compInfo.numOfBlocks * sizeof(SCompBlock)); read(pQuery->hfd, &chksum, sizeof(TSCKSUM)); - if (chksum != taosCalcChecksum(0, (uint8_t *)(pQuery->pBlock), - compInfo.numOfBlocks * sizeof(SCompBlock))) { - dError("vid:%d sid:%d id:%s, head file comp block broken, fileId: %d", - pObj->vnode, pObj->sid, pObj->meterId, pQuery->fileId); - taosLogError("vid:%d sid:%d id:%s, head file comp block broken, fileId: %d", - pObj->vnode, pObj->sid, pObj->meterId, pQuery->fileId); - return -TSDB_CODE_FILE_CORRUPTED; + if (chksum != taosCalcChecksum(0, (uint8_t *)(pQuery->pBlock), compInfo.numOfBlocks * sizeof(SCompBlock))) { + dError("vid:%d sid:%d id:%s, head file comp block broken, fileId: %d", pObj->vnode, pObj->sid, pObj->meterId, + pQuery->fileId); + taosLogError("vid:%d sid:%d id:%s, head file comp block broken, fileId: %d", pObj->vnode, pObj->sid, pObj->meterId, + pQuery->fileId); + return vnodeRecoverFromPeer(pVnode, pQuery->fileId); } close(pQuery->hfd); @@ -971,32 +1011,30 @@ int vnodeGetCompBlockInfo(SMeterObj *pObj, SQuery *pQuery) { sprintf(fileName, "%s.data", prefix); if (stat(fileName, &fstat) < 0) { - dError("vid:%d sid:%d id:%s, data file:%s not there!", pObj->vnode, - pObj->sid, pObj->meterId, fileName); - return -TSDB_CODE_FILE_CORRUPTED; + dError("vid:%d sid:%d id:%s, data file:%s not there!", pObj->vnode, pObj->sid, pObj->meterId, fileName); + return vnodeRecoverFromPeer(pVnode, pQuery->fileId); } if (pQuery->dfd > 0) close(pQuery->dfd); pQuery->dfd = open(fileName, O_RDONLY); if (pQuery->dfd < 0) { - dError("vid:%d sid:%d id:%s, failed to open data file:%s, reason:%s", - pObj->vnode, pObj->sid, pObj->meterId, fileName, strerror(errno)); - return -TSDB_CODE_FILE_CORRUPTED; + dError("vid:%d sid:%d id:%s, failed to open data file:%s, reason:%s", pObj->vnode, pObj->sid, pObj->meterId, + fileName, strerror(errno)); + return vnodeRecoverFromPeer(pVnode, pQuery->fileId); } sprintf(fileName, "%s.last", prefix); if (stat(fileName, &fstat) < 0) { - dError("vid:%d sid:%d id:%s, last file:%s not there!", pObj->vnode, - pObj->sid, pObj->meterId, fileName); - return -TSDB_CODE_FILE_CORRUPTED; + dError("vid:%d sid:%d id:%s, last file:%s not there!", pObj->vnode, pObj->sid, pObj->meterId, fileName); + return vnodeRecoverFromPeer(pVnode, pQuery->fileId); } if (pQuery->lfd > 0) close(pQuery->lfd); pQuery->lfd = open(fileName, O_RDONLY); if (pQuery->lfd < 0) { - dError("vid:%d sid:%d id:%s, failed to open last file:%s, reason:%s", - pObj->vnode, pObj->sid, pObj->meterId, fileName, strerror(errno)); - return -TSDB_CODE_FILE_CORRUPTED; + dError("vid:%d sid:%d id:%s, failed to open last file:%s, reason:%s", pObj->vnode, pObj->sid, pObj->meterId, + fileName, strerror(errno)); + return vnodeRecoverFromPeer(pVnode, pQuery->fileId); } return pQuery->numOfBlocks; @@ -1093,12 +1131,11 @@ int vnodeReadCompBlockToMem(SMeterObj *pObj, SQuery *pQuery, SData *sdata[]) { if (code < 0) goto _over; while (col < pBlock->numOfCols && i < pQuery->numOfCols) { - SColumnFilterMsg *pColFilterMsg = &pQuery->colList[i].data; - if ((*pFields)[col].colId < pColFilterMsg->colId) { + SColumnInfo *pColumnInfo = &pQuery->colList[i].data; + if ((*pFields)[col].colId < pColumnInfo->colId) { ++col; - } else if ((*pFields)[col].colId == pColFilterMsg->colId) { - code = vnodeReadColumnToMem(dfd, pBlock, pFields, col, sdata[i]->data, pColFilterMsg->bytes*pBlock->numOfPoints, - temp, buffer, bufferSize); + } else if ((*pFields)[col].colId == pColumnInfo->colId) { + code = vnodeReadColumnToMem(dfd, pBlock, pFields, col, sdata[i]->data, pColumnInfo->bytes*pBlock->numOfPoints, temp, buffer, bufferSize); if (code < 0) goto _over; ++i; ++col; @@ -1131,7 +1168,7 @@ int vnodeReadCompBlockToMem(SMeterObj *pObj, SQuery *pQuery, SData *sdata[]) { _over: tfree(buffer); tfree(temp); - if ( code < 0 ) code = -TSDB_CODE_FILE_CORRUPTED; + if (code < 0) code = vnodeRecoverFromPeer(vnodeList + pObj->vnode, pQuery->fileId); return code; } @@ -1159,6 +1196,7 @@ int vnodeReadLastBlockToMem(SMeterObj *pObj, SCompBlock *pBlock, SData *sdata[]) tfree(buffer); tfree(temp); tfree(pFields); + if (code < 0) code = vnodeRecoverFromPeer(pVnode, pVnode->fileId); return code; } @@ -1217,7 +1255,7 @@ int vnodeWriteBlockToFile(SMeterObj *pObj, SCompBlock *pCompBlock, SData *data[] } getStatistics(data[0]->data, data[i]->data, pObj->schema[i].bytes, points, pObj->schema[i].type, &fields[i].min, - &fields[i].max, &fields[i].sum, &fields[i].wsum, &fields[i].numOfNullPoints); + &fields[i].max, &fields[i].sum, &fields[i].minIndex, &fields[i].maxIndex, &fields[i].numOfNullPoints); } tfree(buffer); @@ -1229,7 +1267,11 @@ int vnodeWriteBlockToFile(SMeterObj *pObj, SCompBlock *pCompBlock, SData *data[] tfree(fields); dError("vid:%d sid:%d id:%s, failed to write block, wlen:%d reason:%s", pObj->vnode, pObj->sid, pObj->meterId, wlen, strerror(errno)); +#ifdef CLUSTER + return vnodeRecoverFromPeer(pVnode, pVnode->commitFileId); +#else return -1; +#endif } pVnode->vnodeStatistic.compStorage += wlen; pVnode->dfSize += wlen; @@ -1247,7 +1289,7 @@ int vnodeWriteBlockToFile(SMeterObj *pObj, SCompBlock *pCompBlock, SData *data[] if (wlen <= 0) { dError("vid:%d sid:%d id:%s, failed to write block, wlen:%d points:%d reason:%s", pObj->vnode, pObj->sid, pObj->meterId, wlen, points, strerror(errno)); - return -TSDB_CODE_FILE_CORRUPTED; + return vnodeRecoverFromPeer(pVnode, pVnode->commitFileId); } pVnode->vnodeStatistic.compStorage += wlen; @@ -1272,7 +1314,7 @@ static int forwardInFile(SQuery *pQuery, int32_t midSlot, int32_t step, SVnodeOb int vnodeSearchPointInFile(SMeterObj *pObj, SQuery *pQuery) { TSKEY latest, oldest; int ret = 0; - long delta = 0; + int64_t delta = 0; int firstSlot, lastSlot, midSlot; int numOfBlocks; char * temp = NULL, *data = NULL; @@ -1290,7 +1332,7 @@ int vnodeSearchPointInFile(SMeterObj *pObj, SQuery *pQuery) { if (pVnode->numOfFiles <= 0) return 0; SVnodeCfg *pCfg = &pVnode->cfg; - delta = (long)pCfg->daysPerFile * tsMsPerDay[pVnode->cfg.precision]; + delta = (int64_t)pCfg->daysPerFile * tsMsPerDay[pVnode->cfg.precision]; latest = pObj->lastKeyOnFile; oldest = (pVnode->fileId - pVnode->numOfFiles + 1) * delta; @@ -1380,7 +1422,7 @@ int vnodeSearchPointInFile(SMeterObj *pObj, SQuery *pQuery) { pObj->pointsPerFileBlock*TSDB_KEYSIZE+EXTRA_BYTES, temp, buffer, bufferSize); if (ret < 0) { - ret = -TSDB_CODE_FILE_CORRUPTED; + ret = vnodeRecoverFromPeer(pVnode, pQuery->fileId); break; } // file broken @@ -1602,8 +1644,8 @@ int vnodeQueryFromFile(SMeterObj *pObj, SQuery *pQuery) { } else { // check each data one by one set the input column data for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { - SColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - pFilterInfo->pData = sdata[pFilterInfo->pFilter.colIdxInBuf]->data; + struct SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; + pFilterInfo->pData = sdata[pFilterInfo->info.colIdxInBuf]->data; } int32_t *ids = calloc(1, numOfReads * sizeof(int32_t)); @@ -1750,8 +1792,11 @@ int vnodeUpdateFileMagic(int vnode, int fileId) { return -1; } +#ifdef CLUSTER + //if (fstat.st_size == size) return 0; +#else if (fstat.st_size == size) return 0; - +#endif vnodeGetHeadDataLname(NULL, fileName, NULL, vnode, fileId); if (stat(fileName, &fstat) == 0) { magic = fstat.st_size; @@ -1817,80 +1862,3 @@ int vnodeRecoverDataFile(int vnode, int fileId) { assert(0); return 0; } - -int vnodeCheckNewHeaderFile(int fd, SVnodeObj *pVnode) { - SCompHeader *pHeader = NULL; - SCompBlock *pBlocks = NULL; - int blockSize = 0; - SCompInfo compInfo; - int tmsize = 0; - - tmsize = sizeof(SCompHeader) * pVnode->cfg.maxSessions + sizeof(TSCKSUM); - - pHeader = (SCompHeader *)malloc(tmsize); - if (pHeader == NULL) return 0; - - lseek(fd, TSDB_FILE_HEADER_LEN, SEEK_SET); - if (read(fd, (void *)pHeader, tmsize) != tmsize) { - goto _broken_exit; - } - - if (!taosCheckChecksumWhole((uint8_t *)pHeader, tmsize)) { - goto _broken_exit; - } - - for (int sid = 0; sid < pVnode->cfg.maxSessions; sid++) { - if (pVnode->meterList == NULL) goto _correct_exit; - if (pVnode->meterList[sid] == NULL || pHeader[sid].compInfoOffset == 0) continue; - lseek(fd, pHeader[sid].compInfoOffset, SEEK_SET); - - if (read(fd, (void *)(&compInfo), sizeof(SCompInfo)) != sizeof(SCompInfo)) { - goto _broken_exit; - } - - if (!taosCheckChecksumWhole((uint8_t *)(&compInfo), sizeof(SCompInfo))) { - goto _broken_exit; - } - - if (compInfo.uid != ((SMeterObj *)pVnode->meterList[sid])->uid) continue; - - int expectedSize = sizeof(SCompBlock) * compInfo.numOfBlocks + sizeof(TSCKSUM); - if (blockSize < expectedSize) { - pBlocks = (SCompBlock *)realloc(pBlocks, expectedSize); - if (pBlocks == NULL) { - tfree(pHeader); - return 0; - } - - blockSize = expectedSize; - } - - if (read(fd, (void *)pBlocks, expectedSize) != expectedSize) { - dError("failed to read block part"); - goto _broken_exit; - } - if (!taosCheckChecksumWhole((uint8_t *)pBlocks, expectedSize)) { - dError("block part is broken"); - goto _broken_exit; - } - - for (int i = 0; i < compInfo.numOfBlocks; i++) { - if (pBlocks[i].last && i != compInfo.numOfBlocks-1) { - dError("last block in middle, block:%d", i); - goto _broken_exit; - } - } - } - -_correct_exit: - dTrace("vid: %d new header file %s is correct", pVnode->vnode, pVnode->nfn); - tfree(pBlocks); - tfree(pHeader); - return 0; - -_broken_exit: - dError("vid: %d new header file %s is broken", pVnode->vnode, pVnode->nfn); - tfree(pBlocks); - tfree(pHeader); - return -1; -} diff --git a/src/system/detail/src/vnodeFileUtil.c b/src/system/detail/src/vnodeFileUtil.c new file mode 100644 index 0000000000..f8de6c4a42 --- /dev/null +++ b/src/system/detail/src/vnodeFileUtil.c @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include +#include +#include +#include +#include + +#include "vnode.h" + +int vnodeCheckHeaderFile(int fd, int dfd, SVnodeCfg cfg, int mode) { + SCompHeader *pHeaders = NULL; + SVnodeCfg *pCfg = &cfg; + SCompInfo compInfo; + SCompBlock *pBlocks = NULL; + int blockSize = 0; + SField *pFields = NULL; + char *pBuf = NULL; + int size = 0; + int ret = 0; + + if (fd < 0 || dfd < 0) return -1; + + lseek(fd, TSDB_FILE_HEADER_LEN, SEEK_SET); + size = pCfg->maxSessions*sizeof(SCompHeader)+sizeof(TSCKSUM); + pHeaders = calloc(1, size); + if (pHeaders == NULL) { + return -1; + } + + read(fd, pHeaders, size); + if (!taosCheckChecksumWhole((uint8_t *)pHeaders, size)) { + return -1; + } + + for (int i = 0; i < pCfg->maxSessions; i++) { + if (pHeaders[i].compInfoOffset == 0) continue; + if (pHeaders[i].compInfoOffset < 0) { + // TODO : report error here + ret = -1; + continue; + } + lseek(fd, pHeaders[i].compInfoOffset, SEEK_SET); + read(fd, &compInfo, sizeof(SCompInfo)); + if (!taosCheckChecksumWhole((uint8_t *)&compInfo, sizeof(SCompInfo))) { + // TODO : report error + ret = -1; + continue; + } + + int tsize = sizeof(SCompBlock) * compInfo.numOfBlocks + sizeof(TSCKSUM); + if (tsize > blockSize) { + if (pBlocks == NULL) { + pBlocks = calloc(1, tsize); + } else { + pBlocks = realloc(pBlocks, tsize); + } + blockSize = tsize; + } + + read(fd, tsize); + if (!taosCheckChecksumWhole(pBlocks, tsize)) { + // TODO: Report error + ret = -1; + continue; + } + + TSKEY keyLast = 0; + for (int j = 0; j < compInfo.numOfBlocks; j++) { + SCompBlock *pBlock = pBlocks + j; + if (pBlock->last != 0 && j < compInfo.numOfBlocks-1) { + // TODO: report error + ret = -1; + break; + } + + if (pBlock->offset < TSDB_FILE_HEADER_LEN) { + // TODO : report erro + ret = -1; + break; + } + + if (pBlock->keyLast < pBlock->keyFirst) { + // TODO : report error + ret = -1; + break; + } + + if (pBlock->keyFirst <= keyLast) { + // TODO : report error + ret = -1; + break; + } + keyLast = pBlock->keyLast; + + // Check block in data + lseek(dfd, pBlock->offset, SEEK_SET); + tsize = sizeof(SField) * pBlock->numOfCols + sizeof(TSCKSUM); + pFields = realloc(pFields, tsize); + + read(dfd, pFields, tsize); + if (!taosCheckChecksumWhole((uint8_t*)pFields, tsize)) { + // TODO : report error + ret = -1; + continue; + } + + for (int k = 0; k < pBlock->numOfCols; k++) { + // TODO: Check pFields[k] content + + pBuf = realloc(pBuf, pFields[k].len); + + if (!taosCheckChecksumWhole((uint8_t *)pBuf, pFields[k].len)) { + // TODO : report error; + ret = -1; + continue; + } + } + } + } + + tfree(pBuf); + tfree(pFields); + tfree(pBlocks); + tfree(pHeaders); + return ret; +} + +int vnodePackDataFile(int vnode, int fileId) { + // TODO: check if it is able to pack current file + + // TODO: assign value to headerFile and dataFile + char *headerFile = NULL; + char *dataFile = NULL; + char *lastFile = NULL; + SVnodeObj *pVnode = vnodeList+vnode; + SCompHeader *pHeaders = NULL; + SCompBlock *pBlocks = NULL; + int blockSize = 0; + char *pBuff = 0; + int buffSize = 0; + SCompInfo compInfo; + int size = 0; + + int hfd = open(headerFile, O_RDONLY); + if (hfd < 0) { + dError("vid: %d, failed to open header file:%s\n", vnode, headerFile); + return -1; + } + int dfd = open(dataFile, O_RDONLY); + if (dfd < 0) { + dError("vid: %d, failed to open data file:%s\n", vnode, dataFile); + return -1; + } + int lfd = open(lastFile, O_RDONLY); + if (lfd < 0) { + dError("vid: %d, failed to open data file:%s\n", vnode, lastFile); + return -1; + } + + lseek(hfd, TSDB_FILE_HEADER_LEN, SEEK_SET); + size = sizeof(SCompHeader)*pVnode->cfg.maxSessions+sizeof(TSCKSUM); + pHeaders = malloc(size); + if (pHeaders == NULL) goto _exit_failure; + read(hfd, pHeaders, size); + if (!taosCheckChecksumWhole((uint8_t *)pHeaders, size)) { + dError("vid: %d, header file %s is broken", vnode, headerFile); + goto _exit_failure; + } + + for (size_t i = 0; i < pVnode->cfg.maxSessions; i++) + { + if (pHeaders[i].compInfoOffset <= 0) continue; + SMeterObj *pObj = (SMeterObj *)pVnode->meterList[i]; + // read compInfo part + lseek(hfd, pHeaders[i].compInfoOffset, SEEK_SET); + read(hfd, &compInfo, sizeof(SCompInfo)); + if (!taosCheckChecksumWhole((uint8_t *)&compInfo, sizeof(SCompInfo))) { + dError("vid: %d sid:%d fileId:%d compInfo is broken", vnode, i, fileId); + goto _exit_failure; + } + + // read compBlock part + int tsize = compInfo.numOfBlocks * sizeof(SCompBlock) + sizeof(TSCKSUM); + if (tsize > blockSize) { + if (blockSize == 0) { + pBlocks = malloc(tsize); + } else { + pBlocks = realloc(pBlocks, tsize); + } + blockSize = tsize; + } + read(hfd, pBlocks, tsize); + if (!taosCheckChecksumWhole((uint8_t *)pBlocks, tsize)) { + dError("vid:%d sid:%d fileId:%d block part is broken", vnode, i, fileId); + goto _exit_failure; + } + + assert(compInfo.numOfBlocks > 0); + // Loop to scan the blocks and merge block when neccessary. + tsize = sizeof(SCompInfo) + compInfo.numOfBlocks *sizeof(SCompBlock) + sizeof(TSCKSUM); + pBuff = realloc(pBuff, tsize); + SCompInfo *pInfo = (SCompInfo *)pBuff; + SCompBlock *pNBlocks = pBuff + sizeof(SCompInfo); + int nCounter = 0; + for (int j; j < compInfo.numOfBlocks; j++) { + // TODO : Check if it is the last block + // if (j == compInfo.numOfBlocks - 1) {} + if (pBlocks[j].numOfPoints + pNBlocks[nCounter].numOfPoints <= pObj->pointsPerFileBlock) { + // Merge current block to current new block + } else { + // Write new block to new data file + // pNBlocks[nCounter]. + nCounter++; + } + } + } + + return 0; + +_exit_failure: + tfree(pHeaders); + if (hfd > 0) close(hfd); + if (dfd > 0) close(dfd); + if (lfd > 0) close(lfd); + return -1; +} \ No newline at end of file diff --git a/src/system/detail/src/vnodeFilterFunc.c b/src/system/detail/src/vnodeFilterFunc.c new file mode 100644 index 0000000000..bd829460d9 --- /dev/null +++ b/src/system/detail/src/vnodeFilterFunc.c @@ -0,0 +1,562 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include +#include +#include +#include +#include + +#include "taosmsg.h" +#include "tsqlfunction.h" +#include "vnode.h" +#include "vnodeDataFilterFunc.h" + +bool less_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval < pFilter->filterInfo.upperBndi); +} + +bool less_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval < pFilter->filterInfo.upperBndi); +} + +bool less_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval < pFilter->filterInfo.upperBndi); +} + +bool less_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval < pFilter->filterInfo.upperBndi); +} + +bool less_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval < pFilter->filterInfo.upperBndd); +} + +bool less_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval < pFilter->filterInfo.upperBndd); +} + +////////////////////////////////////////////////////////////////// +bool large_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +bool large_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +bool large_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +bool large_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +bool large_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)maxval > pFilter->filterInfo.lowerBndd); +} + +bool large_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)maxval > pFilter->filterInfo.lowerBndd); +} +///////////////////////////////////////////////////////////////////// + +bool lessEqual_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi); +} + +bool lessEqual_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi); +} + +bool lessEqual_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi); +} + +bool lessEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi); +} + +bool lessEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval <= pFilter->filterInfo.upperBndd); +} + +bool lessEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval <= pFilter->filterInfo.upperBndd); +} + +////////////////////////////////////////////////////////////////////////// +bool largeEqual_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool largeEqual_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool largeEqual_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool largeEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool largeEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool largeEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)maxval >= pFilter->filterInfo.lowerBndd); +} + +//////////////////////////////////////////////////////////////////////// + +bool equal_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int8_t *)minval == *(int8_t *)maxval) { + return (*(int8_t *)minval == pFilter->filterInfo.lowerBndi); + } else { /* range filter */ + assert(*(int8_t *)minval < *(int8_t *)maxval); + + return *(int8_t *)minval <= pFilter->filterInfo.lowerBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int16_t *)minval == *(int16_t *)maxval) { + return (*(int16_t *)minval == pFilter->filterInfo.lowerBndi); + } else { /* range filter */ + assert(*(int16_t *)minval < *(int16_t *)maxval); + + return *(int16_t *)minval <= pFilter->filterInfo.lowerBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int32_t *)minval == *(int32_t *)maxval) { + return (*(int32_t *)minval == pFilter->filterInfo.lowerBndi); + } else { /* range filter */ + assert(*(int32_t *)minval < *(int32_t *)maxval); + + return *(int32_t *)minval <= pFilter->filterInfo.lowerBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int64_t *)minval == *(int64_t *)maxval) { + return (*(int64_t *)minval == pFilter->filterInfo.lowerBndi); + } else { /* range filter */ + assert(*(int64_t *)minval < *(int64_t *)maxval); + + return *(int64_t *)minval <= pFilter->filterInfo.lowerBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(float *)minval == *(float *)maxval) { + return (fabs(*(float *)minval - pFilter->filterInfo.lowerBndd) <= FLT_EPSILON); + } else { /* range filter */ + assert(*(float *)minval < *(float *)maxval); + return *(float *)minval <= pFilter->filterInfo.lowerBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd; + } +} + +bool equal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(double *)minval == *(double *)maxval) { + return (*(double *)minval == pFilter->filterInfo.lowerBndd); + } else { /* range filter */ + assert(*(double *)minval < *(double *)maxval); + + return *(double *)minval <= pFilter->filterInfo.lowerBndi && *(double *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { + // query condition string is greater than the max length of string, not qualified data + if (pFilter->filterInfo.len > pFilter->bytes) { + return false; + } + + return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) == 0; +} + +bool equal_nchar(SColumnFilterElem *pFilter, char *minval, char *maxval) { + // query condition string is greater than the max length of string, not qualified data + if (pFilter->filterInfo.len > pFilter->bytes) { + return false; + } + + return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE) == 0; +} + +//////////////////////////////////////////////////////////////// +bool like_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + + return patternMatch((char *)pFilter->filterInfo.pz, minval, pFilter->bytes, &info) == TSDB_PATTERN_MATCH; +} + +bool like_nchar(SColumnFilterElem* pFilter, char* minval, char *maxval) { + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + + return WCSPatternMatch((wchar_t*) pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH; +} + +//////////////////////////////////////////////////////////////// +/** + * If minval equals to maxval, it may serve as the one element filter, + * or all elements of an array are identical during pref-filter stage. + * Otherwise, it must be pre-filter of array list of elements. + * + * During pre-filter stage, if there is one element that locates in [minval, maxval], + * the filter function will return true. + */ +bool nequal_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int8_t *)minval == *(int8_t *)maxval) { + return (*(int8_t *)minval != pFilter->filterInfo.lowerBndi); + } + + return true; +} + +bool nequal_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int16_t *)minval == *(int16_t *)maxval) { + return (*(int16_t *)minval != pFilter->filterInfo.lowerBndi); + } + + return true; +} + +bool nequal_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int32_t *)minval == *(int32_t *)maxval) { + return (*(int32_t *)minval != pFilter->filterInfo.lowerBndi); + } + + return true; +} + +bool nequal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int64_t *)minval == *(int64_t *)maxval) { + return (*(int64_t *)minval != pFilter->filterInfo.lowerBndi); + } + + return true; +} + +bool nequal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(float *)minval == *(float *)maxval) { + return (*(float *)minval != pFilter->filterInfo.lowerBndd); + } + + return true; +} + +bool nequal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(double *)minval == *(double *)maxval) { + return (*(double *)minval != pFilter->filterInfo.lowerBndd); + } + + return true; +} + +bool nequal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (pFilter->filterInfo.len > pFilter->bytes) { + return true; + } + + return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) != 0; +} + +bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { + if (pFilter->filterInfo.len > pFilter->bytes) { + return true; + } + + return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*)minval, pFilter->bytes/TSDB_NCHAR_SIZE) != 0; +} + +//////////////////////////////////////////////////////////////// + +bool rangeFilter_i32_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i32_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minvalfilterInfo.upperBndi &&*(int32_t *)maxval> pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i32_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval < pFilter->filterInfo.upperBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i32_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi && *(int32_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +/////////////////////////////////////////////////////////////////////////////// +bool rangeFilter_i8_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i8_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minvalfilterInfo.upperBndi &&*(int8_t *)maxval> pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i8_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval < pFilter->filterInfo.upperBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i8_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi && *(int8_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +///////////////////////////////////////////////////////////////////////////////////// +bool rangeFilter_i16_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i16_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minvalfilterInfo.upperBndi &&*(int16_t *)maxval> pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i16_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval < pFilter->filterInfo.upperBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i16_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi && *(int16_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +//////////////////////////////////////////////////////////////////////// +bool rangeFilter_i64_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i64_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minvalfilterInfo.upperBndi &&*(int64_t *)maxval> pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i64_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval < pFilter->filterInfo.upperBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i64_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi && *(int64_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +//////////////////////////////////////////////////////////////////////// +bool rangeFilter_ds_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_ds_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minvalfilterInfo.upperBndd &&*(float *)maxval> pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_ds_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval < pFilter->filterInfo.upperBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_ds_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval > pFilter->filterInfo.lowerBndd); +} + +////////////////////////////////////////////////////////////////////////// +bool rangeFilter_dd_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval <= pFilter->filterInfo.upperBndd && *(double *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_dd_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minvalfilterInfo.upperBndd &&*(double *)maxval> pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_dd_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval < pFilter->filterInfo.upperBndd && *(double *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_dd_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval <= pFilter->filterInfo.upperBndd && *(double *)maxval > pFilter->filterInfo.lowerBndd); +} + +//////////////////////////////////////////////////////////////////////////// +bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_i8, + large_i8, + equal_i8, + lessEqual_i8, + largeEqual_i8, + nequal_i8, + NULL, +}; + +bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_i16, + large_i16, + equal_i16, + lessEqual_i16, + largeEqual_i16, + nequal_i16, + NULL, +}; + +bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_i32, + large_i32, + equal_i32, + lessEqual_i32, + largeEqual_i32, + nequal_i32, + NULL, +}; + +bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_i64, + large_i64, + equal_i64, + lessEqual_i64, + largeEqual_i64, + nequal_i64, + NULL, +}; + +bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_ds, + large_ds, + equal_ds, + lessEqual_ds, + largeEqual_ds, + nequal_ds, + NULL, +}; + +bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_dd, + large_dd, + equal_dd, + lessEqual_dd, + largeEqual_dd, + nequal_dd, + NULL, +}; + +bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval) = { + NULL, + NULL, + NULL, + equal_str, + NULL, + NULL, + nequal_str, + like_str, +}; + +bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxval) = { + NULL, + NULL, + NULL, + equal_nchar, + NULL, + NULL, + nequal_nchar, + like_nchar, +}; + +bool (*rangeFilterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_i8_ee, + rangeFilter_i8_ie, + rangeFilter_i8_ei, + rangeFilter_i8_ii, +}; + +bool (*rangeFilterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_i16_ee, + rangeFilter_i16_ie, + rangeFilter_i16_ei, + rangeFilter_i16_ii, +}; + +bool (*rangeFilterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_i32_ee, + rangeFilter_i32_ie, + rangeFilter_i32_ei, + rangeFilter_i32_ii, +}; + +bool (*rangeFilterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_i64_ee, + rangeFilter_i64_ie, + rangeFilter_i64_ei, + rangeFilter_i64_ii, +}; + +bool (*rangeFilterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_ds_ee, + rangeFilter_ds_ie, + rangeFilter_ds_ei, + rangeFilter_ds_ii, +}; + +bool (*rangeFilterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_dd_ee, + rangeFilter_dd_ie, + rangeFilter_dd_ei, + rangeFilter_dd_ii, +}; + +__filter_func_t* vnodeGetRangeFilterFuncArray(int32_t type) { + switch(type) { + case TSDB_DATA_TYPE_BOOL: return rangeFilterFunc_i8; + case TSDB_DATA_TYPE_TINYINT: return rangeFilterFunc_i8; + case TSDB_DATA_TYPE_SMALLINT: return rangeFilterFunc_i16; + case TSDB_DATA_TYPE_INT: return rangeFilterFunc_i32; + case TSDB_DATA_TYPE_TIMESTAMP: //timestamp uses bigint filter + case TSDB_DATA_TYPE_BIGINT: return rangeFilterFunc_i64; + case TSDB_DATA_TYPE_FLOAT: return rangeFilterFunc_ds; + case TSDB_DATA_TYPE_DOUBLE: return rangeFilterFunc_dd; + default:return NULL; + } +} + +__filter_func_t* vnodeGetValueFilterFuncArray(int32_t type) { + switch(type) { + case TSDB_DATA_TYPE_BOOL: return filterFunc_i8; + case TSDB_DATA_TYPE_TINYINT: return filterFunc_i8; + case TSDB_DATA_TYPE_SMALLINT: return filterFunc_i16; + case TSDB_DATA_TYPE_INT: return filterFunc_i32; + case TSDB_DATA_TYPE_TIMESTAMP: //timestamp uses bigint filter + case TSDB_DATA_TYPE_BIGINT: return filterFunc_i64; + case TSDB_DATA_TYPE_FLOAT: return filterFunc_ds; + case TSDB_DATA_TYPE_DOUBLE: return filterFunc_dd; + case TSDB_DATA_TYPE_BINARY: return filterFunc_str; + case TSDB_DATA_TYPE_NCHAR: return filterFunc_nchar; + default: return NULL; + } +} + +bool vnodeSupportPrefilter(int32_t type) { return type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR; } diff --git a/src/system/src/vnodeImport.c b/src/system/detail/src/vnodeImport.c similarity index 96% rename from src/system/src/vnodeImport.c rename to src/system/detail/src/vnodeImport.c index 9691206bf9..f50b6f4946 100644 --- a/src/system/src/vnodeImport.c +++ b/src/system/detail/src/vnodeImport.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include #include @@ -40,7 +41,7 @@ typedef struct { } SHeadInfo; typedef struct { - void *signature; + void * signature; SShellObj *pShell; SMeterObj *pObj; int retry; @@ -60,7 +61,7 @@ typedef struct { SData *sdata[TSDB_MAX_COLUMNS]; char *buffer; char *payload; - char *opayload; // allocated space for payload from client + char *opayload; int rows; } SImportInfo; @@ -203,7 +204,7 @@ int vnodeOpenFileForImport(SImportInfo *pImport, char *payload, SHeadInfo *pHinf return -1; } - if (pHinfo->compInfo.uid == pObj->uid) { + if (pHinfo->compInfo.uid == pObj->uid) { pHinfo->compInfoOffset = pHinfo->headList[pObj->sid].compInfoOffset; pHinfo->leftOffset = pHinfo->headList[pObj->sid].compInfoOffset + sizeof(SCompInfo); } else { @@ -242,7 +243,6 @@ int vnodeOpenFileForImport(SImportInfo *pImport, char *payload, SHeadInfo *pHinf } else if (pVnode->commitFileId == pImport->fileId) { int slots = pImport->pos ? pImport->slot + 1 : pImport->slot; - assert(slots >= 0); pHinfo->leftOffset += slots * sizeof(SCompBlock); // check if last block is at last file, if it is, read into memory @@ -352,7 +352,7 @@ int vnodeImportToFile(SImportInfo *pImport) { SHeadInfo headInfo; int code = 0, col; SCompBlock compBlock; - char *payload = pImport->payload; + char * payload = pImport->payload; int rows = pImport->rows; SCachePool *pPool = (SCachePool *)pVnode->pCachePool; @@ -362,12 +362,12 @@ int vnodeImportToFile(SImportInfo *pImport) { headInfo.headList = malloc(sizeof(SCompHeader) * pCfg->maxSessions + sizeof(TSCKSUM)); SData *cdata[TSDB_MAX_COLUMNS]; - char * buffer1 = + char *buffer1 = malloc(pObj->bytesPerPoint * pCfg->rowsInFileBlock + (sizeof(SData) + EXTRA_BYTES) * pObj->numOfColumns); cdata[0] = (SData *)buffer1; SData *data[TSDB_MAX_COLUMNS]; - char * buffer2 = + char *buffer2 = malloc(pObj->bytesPerPoint * pCfg->rowsInFileBlock + (sizeof(SData) + EXTRA_BYTES) * pObj->numOfColumns); data[0] = (SData *)buffer2; @@ -638,7 +638,7 @@ int vnodeFindKeyInFile(SImportInfo *pImport, int order) { SVnodeObj *pVnode = &vnodeList[pObj->vnode]; int code = -1; SQuery query; - SColumnFilter colList[TSDB_MAX_COLUMNS] = {0}; + SColumnInfoEx colList[TSDB_MAX_COLUMNS] = {0}; TSKEY key = order ? pImport->firstKey : pImport->lastKey; memset(&query, 0, sizeof(query)); @@ -794,8 +794,6 @@ int vnodeImportStartToFile(SImportInfo *pImport, char *payload, int rows) { code = vnodeFindKeyInFile(pImport, 1); if (code != 0) return code; - assert(pImport->slot >= 0); - if (pImport->key != pImport->firstKey) { pImport->payload = payload; pImport->rows = vnodeGetImportStartPart(pObj, payload, rows, pImport->key); @@ -879,16 +877,24 @@ int vnodeImportPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi } payload = pSubmit->payLoad; - int firstId = (*(TSKEY *)payload)/pVnode->cfg.daysPerFile/tsMsPerDay[pVnode->cfg.precision]; - int lastId = (*(TSKEY *)(payload+pObj->bytesPerPoint*(rows-1)))/pVnode->cfg.daysPerFile/tsMsPerDay[pVnode->cfg.precision]; - int cfile = now/pVnode->cfg.daysPerFile/tsMsPerDay[pVnode->cfg.precision]; - if ((firstId <= cfile - pVnode->maxFiles) || (firstId > cfile + 1) || (lastId <= cfile - pVnode->maxFiles) || (lastId > cfile + 1)) { - dError("vid:%d sid:%d id:%s, invalid timestamp to import, rows:%d firstKey: %ld lastKey: %ld", - pObj->vnode, pObj->sid, pObj->meterId, rows, *(TSKEY *)(payload), *(TSKEY *)(payload+pObj->bytesPerPoint*(rows-1))); + TSKEY firstKey = *(TSKEY *)payload; + TSKEY lastKey = *(TSKEY *)(payload + pObj->bytesPerPoint*(rows-1)); + int cfid = now/pVnode->cfg.daysPerFile/tsMsPerDay[pVnode->cfg.precision]; + TSKEY minAllowedKey = (cfid - pVnode->maxFiles + 1)*pVnode->cfg.daysPerFile*tsMsPerDay[pVnode->cfg.precision]; + TSKEY maxAllowedKey = (cfid + 2)*pVnode->cfg.daysPerFile*tsMsPerDay[pVnode->cfg.precision] - 1; + if (firstKey < minAllowedKey || firstKey > maxAllowedKey || lastKey < minAllowedKey || lastKey > maxAllowedKey) { + dError("vid:%d sid:%d id:%s, vnode lastKeyOnFile:%lld, data is out of range, rows:%d firstKey:%lld lastKey:%lld minAllowedKey:%lld maxAllowedKey:%lld", + pObj->vnode, pObj->sid, pObj->meterId, pVnode->lastKeyOnFile, rows, firstKey, lastKey, minAllowedKey, maxAllowedKey); return TSDB_CODE_TIMESTAMP_OUT_OF_RANGE; } - if ( pVnode->cfg.commitLog && source != TSDB_DATA_SOURCE_LOG) { + // forward to peers + if (pShell && pVnode->cfg.replications > 1) { + code = vnodeForwardToPeer(pObj, cont, contLen, TSDB_ACTION_IMPORT, sversion); + if (code != 0) return code; + } + + if (pVnode->cfg.commitLog && source != TSDB_DATA_SOURCE_LOG) { if (pVnode->logFd < 0) return TSDB_CODE_INVALID_COMMIT_LOG; code = vnodeWriteToCommitLog(pObj, TSDB_ACTION_IMPORT, cont, contLen, sversion); if (code != 0) return code; diff --git a/src/system/src/vnodeMeter.c b/src/system/detail/src/vnodeMeter.c similarity index 91% rename from src/system/src/vnodeMeter.c rename to src/system/detail/src/vnodeMeter.c index dfa2197e4c..a595e8f689 100644 --- a/src/system/src/vnodeMeter.c +++ b/src/system/detail/src/vnodeMeter.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include #include @@ -61,8 +62,14 @@ int vnodeUpdateVnodeStatistic(FILE *fp, SVnodeObj *pVnode) { void vnodeUpdateVnodeFileHeader(FILE *fp, SVnodeObj *pVnode) { fseek(fp, TSDB_FILE_HEADER_LEN * 1 / 4, SEEK_SET); + +#ifdef _TD_ARM_32_ + fprintf(fp, "%lld %lld %lld ", pVnode->lastCreate, pVnode->lastRemove, pVnode->version); + fprintf(fp, "%lld %d %d ", pVnode->lastKeyOnFile, pVnode->fileId, pVnode->numOfFiles); +#else fprintf(fp, "%ld %ld %ld ", pVnode->lastCreate, pVnode->lastRemove, pVnode->version); fprintf(fp, "%ld %d %d ", pVnode->lastKeyOnFile, pVnode->fileId, pVnode->numOfFiles); +#endif } int vnodeCreateMeterObjFile(int vnode) { @@ -309,13 +316,27 @@ int vnodeRestoreMeterObj(char *buffer, int64_t length) { dError("vid:%d sid:%d, no memory to allocate", pSavedObj->vnode, pSavedObj->sid); return TSDB_CODE_SERV_OUT_OF_MEMORY; } + + pObj->schema = (SColumn *)malloc(pSavedObj->numOfColumns * sizeof(SColumn)); + if (NULL == pObj->schema){ + dError("vid:%d sid:%d, no memory to allocate for schema", pSavedObj->vnode, pSavedObj->sid); + free(pObj); + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } memcpy(pObj, pSavedObj, offsetof(SMeterObj, reserved)); - vnodeList[pSavedObj->vnode].meterList[pSavedObj->sid] = pObj; pObj->numOfQueries = 0; pObj->pCache = vnodeAllocateCacheInfo(pObj); + if (NULL == pObj->pCache){ + dError("vid:%d sid:%d, no memory to allocate for cache", pSavedObj->vnode, pSavedObj->sid); + tfree(pObj->schema); + tfree(pObj); + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } + + vnodeList[pSavedObj->vnode].meterList[pSavedObj->sid] = pObj; pObj->pStream = NULL; - pObj->schema = (SColumn *)malloc(pSavedObj->numOfColumns * sizeof(SColumn)); + memcpy(pObj->schema, buffer + offsetof(SMeterObj, reserved), pSavedObj->numOfColumns * sizeof(SColumn)); pObj->state = TSDB_METER_STATE_READY; @@ -348,8 +369,13 @@ int vnodeOpenMetersVnode(int vnode) { fread(&(pVnode->vnodeStatistic), sizeof(SVnodeStatisticInfo), 1, fp); fseek(fp, TSDB_FILE_HEADER_LEN * 1 / 4, SEEK_SET); +#ifdef _TD_ARM_32_ + fscanf(fp, "%lld %lld %lld ", &(pVnode->lastCreate), &(pVnode->lastRemove), &(pVnode->version)); + fscanf(fp, "%lld %d %d ", &(pVnode->lastKeyOnFile), &(pVnode->fileId), &(pVnode->numOfFiles)); +#else fscanf(fp, "%ld %ld %ld ", &(pVnode->lastCreate), &(pVnode->lastRemove), &(pVnode->version)); fscanf(fp, "%ld %d %d ", &(pVnode->lastKeyOnFile), &(pVnode->fileId), &(pVnode->numOfFiles)); +#endif fseek(fp, TSDB_FILE_HEADER_LEN * 2 / 4, SEEK_SET); fread(&pVnode->cfg, sizeof(SVnodeCfg), 1, fp); @@ -512,7 +538,6 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi SSubmitMsg *pSubmit = (SSubmitMsg *)cont; char * pData; TSKEY tsKey; - int cfile; int points = 0; int code = TSDB_CODE_SUCCESS; SVnodeObj * pVnode = vnodeList + pObj->vnode; @@ -529,7 +554,6 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi // to guarantee time stamp is the same for all vnodes pData = pSubmit->payLoad; tsKey = now; - cfile = tsKey/pVnode->cfg.daysPerFile/tsMsPerDay[pVnode->cfg.precision]; if (*((TSKEY *)pData) == 0) { for (i = 0; i < numOfPoints; ++i) { *((TSKEY *)pData) = tsKey++; @@ -561,6 +585,11 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi if (code != 0) return code; } + if (source == TSDB_DATA_SOURCE_SHELL && pVnode->cfg.replications > 1) { + code = vnodeForwardToPeer(pObj, cont, contLen, TSDB_ACTION_INSERT, sversion); + if (code != 0) return code; + } + if (pObj->sversion < sversion) { dTrace("vid:%d sid:%d id:%s, schema is changed, new:%d old:%d", pObj->vnode, pObj->sid, pObj->meterId, sversion, pObj->sversion); @@ -573,11 +602,13 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi code = 0; TSKEY firstKey = *((TSKEY *)pData); - int firstId = firstKey/pVnode->cfg.daysPerFile/tsMsPerDay[pVnode->cfg.precision]; - int lastId = (*(TSKEY *)(pData + pObj->bytesPerPoint * (numOfPoints - 1)))/pVnode->cfg.daysPerFile/tsMsPerDay[pVnode->cfg.precision]; - if ((firstId <= cfile - pVnode->maxFiles) || (firstId > cfile + 1) || (lastId <= cfile - pVnode->maxFiles) || (lastId > cfile + 1)) { - dError("vid:%d sid:%d id:%s, invalid timestamp to insert, numOfPoints:%d firstKey: %ld lastKey: %ld ", pObj->vnode, pObj->sid, - pObj->meterId, numOfPoints, firstKey, (*(TSKEY *)(pData + pObj->bytesPerPoint * (numOfPoints - 1)))); + TSKEY lastKey = *((TSKEY *)(pData + pObj->bytesPerPoint * (numOfPoints - 1))); + int cfid = now/pVnode->cfg.daysPerFile/tsMsPerDay[pVnode->cfg.precision]; + TSKEY minAllowedKey = (cfid - pVnode->maxFiles + 1)*pVnode->cfg.daysPerFile*tsMsPerDay[pVnode->cfg.precision]; + TSKEY maxAllowedKey = (cfid + 2)*pVnode->cfg.daysPerFile*tsMsPerDay[pVnode->cfg.precision] - 2; + if (firstKey < minAllowedKey || firstKey > maxAllowedKey || lastKey < minAllowedKey || lastKey > maxAllowedKey) { + dError("vid:%d sid:%d id:%s, vnode lastKeyOnFile:%lld, data is out of range, numOfPoints:%d firstKey:%lld lastKey:%lld minAllowedKey:%lld maxAllowedKey:%lld", + pObj->vnode, pObj->sid, pObj->meterId, pVnode->lastKeyOnFile, numOfPoints,firstKey, lastKey, minAllowedKey, maxAllowedKey); return TSDB_CODE_TIMESTAMP_OUT_OF_RANGE; } @@ -634,15 +665,27 @@ _over: return code; } +/** + * continue running of the function may cause the free vnode crash with high probability + * todo fix it by set flag to disable commit in any cases + * + * @param param + * @param tmrId + */ void vnodeProcessUpdateSchemaTimer(void *param, void *tmrId) { SMeterObj * pObj = (SMeterObj *)param; - if (pObj->vnode >= TSDB_MAX_VNODES) return; SVnodeObj * pVnode = vnodeList + pObj->vnode; - if (pVnode == NULL) return; + + /* + * vnode may have been dropped, check it in the first place + * if the vnode is freed, the pObj is not valid any more, the pObj->vnode is meanless + * so may be the vid should be passed into this function as a parameter? + */ if (pVnode->meterList == NULL) { dTrace("vnode is deleted, abort update schema"); return; } + SCachePool *pPool = (SCachePool *)pVnode->pCachePool; pthread_mutex_lock(&pPool->vmutex); @@ -698,7 +741,6 @@ void vnodeUpdateMeter(void *param, void *tmrId) { if (num > 0 || state != TSDB_METER_STATE_READY) { // the state may have been changed by vnodeSetMeterState, recover it in the first place vnodeClearMeterState(pObj, TSDB_METER_STATE_UPDATING); - dTrace("vid:%d sid:%d id:%s, update failed, retry later, numOfQueries:%d, state:%d", pNew->vnode, pNew->sid, pNew->meterId, num, state); diff --git a/src/system/src/vnodeQueryImpl.c b/src/system/detail/src/vnodeQueryImpl.c similarity index 75% rename from src/system/src/vnodeQueryImpl.c rename to src/system/detail/src/vnodeQueryImpl.c index be004e95e9..ac31cd31c2 100644 --- a/src/system/src/vnodeQueryImpl.c +++ b/src/system/detail/src/vnodeQueryImpl.c @@ -13,22 +13,13 @@ * along with this program. If not, see . */ -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include - +#include "os.h" #include "taosmsg.h" #include "textbuffer.h" +#include "ttime.h" + #include "tinterpolation.h" +#include "tscJoinProcess.h" #include "tscSecondaryMerge.h" #include "tscompression.h" #include "ttime.h" @@ -41,20 +32,21 @@ #include "vnodeFile.h" #include "vnodeQueryImpl.h" +enum { + TS_JOIN_TS_EQUAL = 0, + TS_JOIN_TS_NOT_EQUALS = 1, + TS_JOIN_TAG_NOT_EQUALS = 2, +}; + +#define IS_DISK_DATA_BLOCK(q) ((q)->fileId >= 0) + static int32_t copyDataFromMMapBuffer(int fd, SQInfo *pQInfo, SQueryFileInfo *pQueryFile, char *buf, uint64_t offset, int32_t size); static int32_t readDataFromDiskFile(int fd, SQInfo *pQInfo, SQueryFileInfo *pQueryFile, char *buf, uint64_t offset, int32_t size); -__read_data_fn_t readDataFunctor[2] = { - copyDataFromMMapBuffer, readDataFromDiskFile, -}; - -#define IS_DISK_DATA_BLOCK(q) ((q)->fileId >= 0) - -static int64_t comp_block_info_read_bytes = 0; +__read_data_fn_t readDataFunctor[2] = {copyDataFromMMapBuffer, readDataFromDiskFile}; -static void destroyMeterQueryInfo(SMeterQueryInfo *pMeterQInfo); static void vnodeInitLoadCompBlockInfo(SQueryLoadCompBlockInfo *pCompBlockLoadInfo); static int32_t moveToNextBlock(SQueryRuntimeEnv *pRuntimeEnv, int32_t step, __block_search_fn_t searchFn, bool loadData); @@ -68,14 +60,14 @@ static TSKEY getTimestampInDiskBlock(SQueryRuntimeEnv *pRuntimeEnv, int32_t inde static void savePointPosition(SPositionInfo *position, int32_t fileId, int32_t slot, int32_t pos); static int32_t getNextDataFileCompInfo(SQueryRuntimeEnv *pRuntimeEnv, SMeterObj *pMeterObj, int32_t step); -static void setOutputBuffer(SQueryRuntimeEnv *pRuntimeEnv, SOutputRes *pResult); +static void setGroupOutputBuffer(SQueryRuntimeEnv *pRuntimeEnv, SOutputRes *pResult); static void getAlignedIntervalQueryRange(SQuery *pQuery, TSKEY keyInData, TSKEY skey, TSKEY ekey); static void doApplyIntervalQueryOnBlock(SMeterQuerySupportObj *pSupporter, SMeterQueryInfo *pInfo, SBlockInfo *pBlockInfo, int64_t *pPrimaryCol, char *sdata, SField *pFields, __block_search_fn_t searchFn); -static void saveResult(SMeterQuerySupportObj *pSupporter, SMeterQueryInfo *sqinfo, int32_t numOfResult); +static void saveResult(SMeterQuerySupportObj *pSupporter, SMeterQueryInfo *pMeterQueryInfo, int32_t numOfResult); static void applyIntervalQueryOnBlock(SMeterQuerySupportObj *pSupporter, SMeterDataInfo *pInfoEx, char *data, int64_t *pPrimaryData, SBlockInfo *pBlockInfo, int32_t blockStatus, SField *pFields, __block_search_fn_t searchFn); @@ -86,6 +78,7 @@ static void flushFromResultBuf(SMeterQuerySupportObj *pSupporter, const SQuery * static void validateTimestampForSupplementResult(SQueryRuntimeEnv *pRuntimeEnv, int64_t numOfIncrementRes); static void getBasicCacheInfoSnapshot(SQuery *pQuery, SCacheInfo *pCacheInfo, int32_t vid); static void getQueryPositionForCacheInvalid(SQueryRuntimeEnv *pRuntimeEnv, __block_search_fn_t searchFn); +static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); // check the offset value integrity static FORCE_INLINE int32_t validateHeaderOffsetSegment(SQInfo *pQInfo, char *filePath, int32_t vid, char *data, @@ -108,9 +101,8 @@ static FORCE_INLINE int32_t getCompHeaderStartPosition(SVnodeCfg *pCfg) { static FORCE_INLINE int32_t validateCompBlockOffset(SQInfo *pQInfo, SMeterObj *pMeterObj, SCompHeader *pCompHeader, SQueryFileInfo *pQueryFileInfo, int32_t headerSize) { if (pCompHeader->compInfoOffset < headerSize || pCompHeader->compInfoOffset > pQueryFileInfo->headFileSize) { - dError("QInfo:%p vid:%d sid:%d id:%s, compInfoOffset:%d is not valid, size:%ld", - pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pCompHeader->compInfoOffset, - pQueryFileInfo->headFileSize); + dError("QInfo:%p vid:%d sid:%d id:%s, compInfoOffset:%d is not valid, size:%ld", pQInfo, pMeterObj->vnode, + pMeterObj->sid, pMeterObj->meterId, pCompHeader->compInfoOffset, pQueryFileInfo->headFileSize); return -1; } @@ -142,6 +134,51 @@ static FORCE_INLINE int32_t validateCompBlockSegment(SQInfo *pQInfo, char *fileP return 0; } +bool isGroupbyNormalCol(SSqlGroupbyExpr *pGroupbyExpr) { + if (pGroupbyExpr == NULL || pGroupbyExpr->numOfGroupCols == 0) { + return false; + } + + for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { + SColIndexEx *pColIndex = &pGroupbyExpr->columnInfo[i]; + if (pColIndex->flag == TSDB_COL_NORMAL) { + /* + * make sure the normal column locates at the second position if tbname exists in group by clause + */ + if (pGroupbyExpr->numOfGroupCols > 1) { + assert(pColIndex->colIdx > 0); + } + + return true; + } + } + + return false; +} + +bool isSelectivityWithTagsQuery(SQuery *pQuery) { + bool hasTags = false; + int32_t numOfSelectivity = 0; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functId = pQuery->pSelectExpr[i].pBase.functionId; + if (functId == TSDB_FUNC_TAG_DUMMY || functId == TSDB_FUNC_TS_DUMMY) { + hasTags = true; + continue; + } + + if ((aAggs[functId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + numOfSelectivity++; + } + } + + if (numOfSelectivity > 0 && hasTags) { + return true; + } + + return false; +} + static void vnodeFreeFieldsEx(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; vnodeFreeFields(pQuery); @@ -152,9 +189,7 @@ static void vnodeFreeFieldsEx(SQueryRuntimeEnv *pRuntimeEnv) { static bool vnodeIsCompBlockInfoLoaded(SQueryRuntimeEnv *pRuntimeEnv, SMeterObj *pMeterObj, int32_t fileIndex) { SQuery *pQuery = pRuntimeEnv->pQuery; - /* - * check if data file header of this table has been loaded into memory, avoid to reloaded comp Block info - */ + // check if data file header of this table has been loaded into memory, avoid to reloaded comp Block info SQueryLoadCompBlockInfo *pLoadCompBlockInfo = &pRuntimeEnv->loadCompBlockInfo; // if vnodeFreeFields is called, the pQuery->pFields is NULL @@ -227,13 +262,13 @@ static int vnodeGetCompBlockInfo(SMeterObj *pMeterObj, SQueryRuntimeEnv *pRuntim int64_t st = taosGetTimestampUs(); if (vnodeIsCompBlockInfoLoaded(pRuntimeEnv, pMeterObj, fileIndex)) { - dTrace("QInfo:%p vid:%d sid:%d id:%s, fileId:%d compBlock info is loaded, not reload", - GET_QINFO_ADDR(pQuery), pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQueryFileInfo->fileID); + dTrace("QInfo:%p vid:%d sid:%d id:%s, fileId:%d compBlock info is loaded, not reload", GET_QINFO_ADDR(pQuery), + pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQueryFileInfo->fileID); return pQuery->numOfBlocks; } - SQueryCostStatistics *pSummary = &pRuntimeEnv->summary; + SQueryCostSummary *pSummary = &pRuntimeEnv->summary; pSummary->readCompInfo++; pSummary->numOfSeek++; @@ -317,9 +352,9 @@ static int vnodeGetCompBlockInfo(SMeterObj *pMeterObj, SQueryRuntimeEnv *pRuntim vnodeSetCompBlockInfoLoaded(pRuntimeEnv, fileIndex, pMeterObj->sid); int64_t et = taosGetTimestampUs(); - qTrace("QInfo:%p vid:%d sid:%d id:%s, fileId:%d, load compblock info, size:%d, elapsed:%f ms", - pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pRuntimeEnv->pHeaderFiles[fileIndex].fileID, - compBlockSize, (et - st) / 1000.0); + qTrace("QInfo:%p vid:%d sid:%d id:%s, fileId:%d, load compblock info, size:%d, elapsed:%f ms", pQInfo, + pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pRuntimeEnv->pHeaderFiles[fileIndex].fileID, + compBlockSize, (et - st) / 1000.0); pSummary->totalCompInfoSize += compBlockSize; pSummary->loadCompInfoUs += (et - st); @@ -344,8 +379,8 @@ static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, int64_t StartQue char *primaryColumnData, int32_t size, int32_t functionId, SField *pField, bool hasNull, int32_t blockStatus, void *param, int32_t scanFlag); -static tFilePage **createInMemGroupResultBuf(SQLFunctionCtx *pCtx, int32_t nOutputCols, int32_t nAlloc); -static void destroyBuf(tFilePage **pBuf, int32_t nOutputCols); +void createGroupResultBuf(SQuery *pQuery, SOutputRes *pOneResult, bool isMetricQuery); +static void destroyGroupResultBuf(SOutputRes *pOneOutputRes, int32_t nOutputCols); static int32_t binarySearchForBlockImpl(SCompBlock *pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { int32_t firstSlot = 0; @@ -378,8 +413,8 @@ static int32_t binarySearchForBlock(SQuery *pQuery, int64_t key) { return binarySearchForBlockImpl(pQuery->pBlock, pQuery->numOfBlocks, key, pQuery->order.order); } +/* unmap previous buffer */ static UNUSED_FUNC int32_t resetMMapWindow(SQueryFileInfo *pQueryFileInfo) { - /* unmap previous buffer */ munmap(pQueryFileInfo->pDataFileData, pQueryFileInfo->defaultMappingSize); pQueryFileInfo->dtFileMappingOffset = 0; @@ -433,8 +468,8 @@ static int32_t moveMMapWindow(SQueryFileInfo *pQueryFileInfo, uint64_t offset) { /* advise kernel the usage of mmaped data */ if (madvise(pQueryFileInfo->pDataFileData, pQueryFileInfo->defaultMappingSize, MADV_SEQUENTIAL) == -1) { - dError("failed to advise kernel the usage of data file:%s, handle:%d, reason:%s", - pQueryFileInfo->dataFilePath, pQueryFileInfo->dataFd, strerror(errno)); + dError("failed to advise kernel the usage of data file:%s, handle:%d, reason:%s", pQueryFileInfo->dataFilePath, + pQueryFileInfo->dataFd, strerror(errno)); } return 0; @@ -537,7 +572,7 @@ static int32_t loadColumnIntoMem(SQuery *pQuery, SQueryFileInfo *pQueryFileInfo, if (pBlock->algorithm) { (*pDecompFunc[pFields[col].type])(tmpBuf, pFields[col].len, pBlock->numOfPoints, sdata->data, - pFields[col].bytes*pBlock->numOfPoints, pBlock->algorithm, buffer, buffersize); + pFields[col].bytes * pBlock->numOfPoints, pBlock->algorithm, buffer, buffersize); } return 0; @@ -556,7 +591,7 @@ static int32_t loadDataBlockFieldsInfo(SQueryRuntimeEnv *pRuntimeEnv, SQueryFile *pField = malloc(size); } - SQueryCostStatistics *pSummary = &pRuntimeEnv->summary; + SQueryCostSummary *pSummary = &pRuntimeEnv->summary; pSummary->totalFieldSize += size; pSummary->readField++; pSummary->numOfSeek++; @@ -572,15 +607,15 @@ static int32_t loadDataBlockFieldsInfo(SQueryRuntimeEnv *pRuntimeEnv, SQueryFile // check fields integrity if (!taosCheckChecksumWhole((uint8_t *)(*pField), size)) { - dLError("QInfo:%p vid:%d sid:%d id:%s, slot:%d, failed to read sfields, file:%s, sfields area broken:%lld", - pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->slot, pQueryFileInfo->dataFilePath, - pBlock->offset); + dLError("QInfo:%p vid:%d sid:%d id:%s, slot:%d, failed to read sfields, file:%s, sfields area broken:%lld", pQInfo, + pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->slot, pQueryFileInfo->dataFilePath, + pBlock->offset); return -1; } int64_t et = taosGetTimestampUs(); - qTrace("QInfo:%p vid:%d sid:%d id:%s, slot:%d, load field info, size:%d, elapsed:%f ms", - pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->slot, size, (et - st) / 1000.0); + qTrace("QInfo:%p vid:%d sid:%d id:%s, slot:%d, load field info, size:%d, elapsed:%f ms", pQInfo, pMeterObj->vnode, + pMeterObj->sid, pMeterObj->meterId, pQuery->slot, size, (et - st) / 1000.0); pSummary->loadFieldUs += (et - st); return 0; @@ -607,8 +642,8 @@ static int32_t loadDataBlockIntoMem(SCompBlock *pBlock, SField **pField, SQueryR if (vnodeIsDatablockLoaded(pRuntimeEnv, pMeterObj, fileIdx)) { dTrace("QInfo:%p vid:%d sid:%d id:%s, data block has been loaded, ts:%d, slot:%d, brange:%lld-%lld, rows:%d", - GET_QINFO_ADDR(pQuery), pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, loadPrimaryCol, pQuery->slot, - pBlock->keyFirst, pBlock->keyLast, pBlock->numOfPoints); + GET_QINFO_ADDR(pQuery), pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, loadPrimaryCol, pQuery->slot, + pBlock->keyFirst, pBlock->keyLast, pBlock->numOfPoints); return 0; } @@ -618,8 +653,8 @@ static int32_t loadDataBlockIntoMem(SCompBlock *pBlock, SField **pField, SQueryR return -1; } - SQueryCostStatistics *pSummary = &pRuntimeEnv->summary; - int32_t columnBytes = 0; + SQueryCostSummary *pSummary = &pRuntimeEnv->summary; + int32_t columnBytes = 0; int64_t st = taosGetTimestampUs(); @@ -628,9 +663,9 @@ static int32_t loadDataBlockIntoMem(SCompBlock *pBlock, SField **pField, SQueryR *primaryTSBuf = sdata[0]; } else { columnBytes += (*pField)[PRIMARYKEY_TIMESTAMP_COL_INDEX].len + sizeof(TSCKSUM); - int32_t ret = loadColumnIntoMem(pQuery, pQueryFileInfo, pBlock, *pField, PRIMARYKEY_TIMESTAMP_COL_INDEX, - *primaryTSBuf, tmpBuf, pRuntimeEnv->secondaryUnzipBuffer, - pRuntimeEnv->internalBufSize); + int32_t ret = + loadColumnIntoMem(pQuery, pQueryFileInfo, pBlock, *pField, PRIMARYKEY_TIMESTAMP_COL_INDEX, *primaryTSBuf, + tmpBuf, pRuntimeEnv->secondaryUnzipBuffer, pRuntimeEnv->unzipBufSize); if (ret != 0) { return -1; } @@ -642,7 +677,7 @@ static int32_t loadDataBlockIntoMem(SCompBlock *pBlock, SField **pField, SQueryR int32_t ret = 0; - // the first round always be 1, the secondary round is determined by queried function + /* the first round always be 1, the secondary round is determined by queried function */ int32_t round = pRuntimeEnv->scanFlag; while (j < pBlock->numOfCols && i < pQuery->numOfCols) { @@ -662,13 +697,13 @@ static int32_t loadDataBlockIntoMem(SCompBlock *pBlock, SField **pField, SQueryR * 3. in case of filter column required, filter columns must be loaded. */ if (pQuery->colList[i].req[round] == 1 || pQuery->colList[i].data.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - /* if data of this column in current block are all null, do NOT read it from disk */ + // if data of this column in current block are all null, do NOT read it from disk if ((*pField)[j].numOfNullPoints == pBlock->numOfPoints) { fillWithNull(pQuery, sdata[i]->data, i, pBlock->numOfPoints); } else { columnBytes += (*pField)[j].len + sizeof(TSCKSUM); ret = loadColumnIntoMem(pQuery, pQueryFileInfo, pBlock, *pField, j, sdata[i], tmpBuf, - pRuntimeEnv->secondaryUnzipBuffer, pRuntimeEnv->internalBufSize); + pRuntimeEnv->secondaryUnzipBuffer, pRuntimeEnv->unzipBufSize); pSummary->numOfSeek++; } @@ -677,7 +712,8 @@ static int32_t loadDataBlockIntoMem(SCompBlock *pBlock, SField **pField, SQueryR ++j; } else { /* - * pQuery->colList[i].colIdx < (*pFields)[j].colId this column is not existed in current block, fill with NULL value + * pQuery->colList[i].colIdx < (*pFields)[j].colId this column is not existed in current block, + * fill with NULL value */ fillWithNull(pQuery, sdata[i]->data, i, pBlock->numOfPoints); @@ -698,8 +734,8 @@ static int32_t loadDataBlockIntoMem(SCompBlock *pBlock, SField **pField, SQueryR int64_t et = taosGetTimestampUs(); qTrace("QInfo:%p vid:%d sid:%d id:%s, slot:%d, load block completed, ts loaded:%d, rec:%d, elapsed:%f ms", - GET_QINFO_ADDR(pQuery), pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->slot, loadPrimaryCol, - pBlock->numOfPoints, (et - st) / 1000.0); + GET_QINFO_ADDR(pQuery), pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->slot, loadPrimaryCol, + pBlock->numOfPoints, (et - st) / 1000.0); pSummary->totalBlockSize += columnBytes; pSummary->loadBlocksUs += (et - st); @@ -797,8 +833,7 @@ static FORCE_INLINE void saveNextAccessPositionInCache(SPositionInfo *position, savePointPosition(position, -1, slotIdx, pos); } -// todo all functions that call this function should check the returned data -// blocks status +// todo all functions that call this function should check the returned data blocks status SCacheBlock *getCacheDataBlock(SMeterObj *pMeterObj, SQuery *pQuery, int32_t slot) { SCacheInfo *pCacheInfo = (SCacheInfo *)pMeterObj->pCache; if (pCacheInfo == NULL || pCacheInfo->cacheBlocks == NULL || slot < 0) { @@ -904,7 +939,7 @@ static bool getQualifiedDataBlock(SMeterObj *pMeterObj, SQueryRuntimeEnv *pRunti } dError("QInfo:%p fileId:%d total numOfBlks:%d blockId:%d into memory failed due to error in disk files", - GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->numOfBlocks, blkIdx); + GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->numOfBlocks, blkIdx); blkIdx += step; } @@ -915,9 +950,7 @@ static bool getQualifiedDataBlock(SMeterObj *pMeterObj, SQueryRuntimeEnv *pRunti SCompBlock *pBlocks = getDiskDataBlock(pQuery, blkIdx); - /* - * search qualified points in blk, according to primary key (timestamp) column - */ + // search qualified points in blk, according to primary key (timestamp) column pQuery->pos = searchFn(primaryColBuffer->data, pBlocks->numOfPoints, key, pQuery->order.order); assert(pQuery->pos >= 0 && pQuery->fileId >= 0 && pQuery->slot >= 0); @@ -941,7 +974,7 @@ static SField *getFieldInfo(SQuery *pQuery, SBlockInfo *pBlockInfo, SField *pFie SColIndexEx *pColIndexEx = &pQuery->pSelectExpr[column].pBase.colInfo; // for a tag column, no corresponding field info - if (pColIndexEx->isTag) { + if (TSDB_COL_IS_TAG(pColIndexEx->flag)) { return NULL; } @@ -966,7 +999,7 @@ static SField *getFieldInfo(SQuery *pQuery, SBlockInfo *pBlockInfo, SField *pFie static bool hasNullVal(SQuery *pQuery, int32_t col, SBlockInfo *pBlockInfo, SField *pFields, bool isDiskFileBlock) { bool ret = true; - if (pQuery->pSelectExpr[col].pBase.colInfo.isTag) { + if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[col].pBase.colInfo.flag)) { ret = false; } else if (isDiskFileBlock) { if (pFields == NULL) { @@ -989,11 +1022,11 @@ static char *doGetDataBlocks(bool isDiskFileBlock, SQueryRuntimeEnv *pRuntimeEnv if (isDiskFileBlock) { pData = doGetDataBlockImpl(data, colIdx, isDiskFileBlock); } else { - SCacheBlock* pCacheBlock = (SCacheBlock*) data; - SMeterObj* pMeter = pRuntimeEnv->pMeterObj; + SCacheBlock *pCacheBlock = (SCacheBlock *)data; + SMeterObj * pMeter = pRuntimeEnv->pMeterObj; if (colIdx < 0 || pMeter->numOfColumns <= colIdx || pMeter->schema[colIdx].colId != colId) { - /* data in cache is not current available, we need fill the data block in null value */ + // data in cache is not current available, we need fill the data block in null value pData = pRuntimeEnv->colDataBuffer[tmpBufIndex]->data; setNullN(pData, type, bytes, pCacheBlock->numOfPoints); } else { @@ -1005,7 +1038,7 @@ static char *doGetDataBlocks(bool isDiskFileBlock, SQueryRuntimeEnv *pRuntimeEnv } static char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, char *data, SArithmeticSupport *sas, int32_t col, - bool isDiskFileBlock) { + int32_t size, bool isDiskFileBlock) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; @@ -1016,17 +1049,18 @@ static char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, char *data, SArithmeti if (functionId == TSDB_FUNC_ARITHM) { sas->pExpr = &pQuery->pSelectExpr[col]; + // set the start offset to be the lowest start position, no matter asc/desc query order if (QUERY_IS_ASC_QUERY(pQuery)) { - pCtx->startOffset = pQuery->pos; // set the offset value + pCtx->startOffset = pQuery->pos; } else { - pCtx->startOffset = pQuery->pos - pCtx->size + 1; + pCtx->startOffset = pQuery->pos - (size - 1); } for (int32_t i = 0; i < pQuery->numOfCols; ++i) { int32_t colIdx = isDiskFileBlock ? pQuery->colList[i].colIdxInBuf : pQuery->colList[i].colIdx; - SColumnFilterMsg *pColMsg = &pQuery->colList[i].data; - char *pData = doGetDataBlocks(isDiskFileBlock, pRuntimeEnv, data, colIdx, pColMsg->colId, pColMsg->type, + SColumnInfo *pColMsg = &pQuery->colList[i].data; + char * pData = doGetDataBlocks(isDiskFileBlock, pRuntimeEnv, data, colIdx, pColMsg->colId, pColMsg->type, pColMsg->bytes, pQuery->colList[i].colIdxInBuf); sas->elemSize[i] = pColMsg->bytes; @@ -1038,7 +1072,7 @@ static char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, char *data, SArithmeti SColIndexEx *pCol = &pQuery->pSelectExpr[col].pBase.colInfo; int32_t colIdx = isDiskFileBlock ? pCol->colIdxInBuf : pCol->colIdx; - if (pCol->isTag) { + if (TSDB_COL_IS_TAG(pCol->flag)) { dataBlock = NULL; } else { /* @@ -1065,58 +1099,65 @@ static char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, char *data, SArithmeti * @return the incremental number of output value, so it maybe 0 for fixed number of query, * such as count/min/max etc. */ -static int32_t applyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t forwardStep, TSKEY *primaryKeyCol, char *data, - SField *pFields, SBlockInfo *pBlockInfo, bool isDiskFileBlock) { +static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t forwardStep, TSKEY *primaryKeyCol, + char *data, SField *pFields, SBlockInfo *pBlockInfo, bool isDiskFileBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; SQuery * pQuery = pRuntimeEnv->pQuery; int64_t prevNumOfRes = getNumOfResult(pRuntimeEnv); - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { - if (pRuntimeEnv->go[k]) { - SField dummyField = {0}; - SArithmeticSupport sas = {0}; + SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); - // todo refactor - int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; +// if (!functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { +// continue; +// } - if (!IS_MASTER_SCAN(pRuntimeEnv) && - !(functionId == TSDB_FUNC_LAST_DST || functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_FIRST || - functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS)) { - continue; - } + SField dummyField = {0}; - bool hasNull = hasNullVal(pQuery, k, pBlockInfo, pFields, isDiskFileBlock); - char *dataBlock = getDataBlocks(pRuntimeEnv, data, &sas, k, isDiskFileBlock); + bool hasNull = hasNullVal(pQuery, k, pBlockInfo, pFields, isDiskFileBlock); + char *dataBlock = getDataBlocks(pRuntimeEnv, data, &sasArray[k], k, forwardStep, isDiskFileBlock); - SField *tpField = NULL; + SField *tpField = NULL; - if (pFields != NULL) { - tpField = getFieldInfo(pQuery, pBlockInfo, pFields, k); - /* - * Field info not exist, the required column is not present in current block, - * so all data must be null value in current block. - */ - if (tpField == NULL) { - tpField = &dummyField; - tpField->numOfNullPoints = (int32_t)forwardStep; - } + if (pFields != NULL) { + tpField = getFieldInfo(pQuery, pBlockInfo, pFields, k); + /* + * Field info not exist, the required column is not present in current block, + * so all data must be null value in current block. + */ + if (tpField == NULL) { + tpField = &dummyField; + tpField->numOfNullPoints = (int32_t)forwardStep; } + } - TSKEY ts = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->skey : pQuery->ekey; + TSKEY ts = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->skey : pQuery->ekey; - int64_t alignedTimestamp = taosGetIntervalStartTimestamp(ts, pQuery->nAggTimeInterval, pQuery->intervalTimeUnit, - pQuery->precision); - setExecParams(pQuery, &pCtx[k], alignedTimestamp, dataBlock, (char *)primaryKeyCol, forwardStep, functionId, - tpField, hasNull, pRuntimeEnv->blockStatus, &sas, pRuntimeEnv->scanFlag); + int64_t alignedTimestamp = taosGetIntervalStartTimestamp(ts, pQuery->nAggTimeInterval, pQuery->intervalTimeUnit, + pQuery->precision); + setExecParams(pQuery, &pCtx[k], alignedTimestamp, dataBlock, (char *)primaryKeyCol, forwardStep, functionId, + tpField, hasNull, pRuntimeEnv->blockStatus, &sasArray[k], pRuntimeEnv->scanFlag); + } - pRuntimeEnv->go[k] = aAggs[functionId].xFunction(&pCtx[k]); + /* + * the sqlfunctionCtx parameters should be set done before all functions are invoked, + * since the selectivity + tag_prj query needs all parameters been set done. + * tag_prj function are changed to be TSDB_FUNC_TAG_DUMMY + */ + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunction(&pCtx[k]); } } int64_t numOfIncrementRes = getNumOfResult(pRuntimeEnv) - prevNumOfRes; validateTimestampForSupplementResult(pRuntimeEnv, numOfIncrementRes); + tfree(sasArray); + return (int32_t)numOfIncrementRes; } @@ -1125,8 +1166,8 @@ static int32_t applyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t forwardS * 1. count(*)/spread(ts) is invoked * 2. this column does not exists * - * first filter the data block according to the value filter condition, then, if - * the top/bottom query applied, invoke the filter function to decide if the data block need to be accessed or not. + * first filter the data block according to the value filter condition, then, if the top/bottom query applied, + * invoke the filter function to decide if the data block need to be accessed or not. * @param pQuery * @param pField * @return @@ -1137,37 +1178,41 @@ static bool needToLoadDataBlock(SQuery *pQuery, SField *pField, SQLFunctionCtx * } for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { - SColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - int32_t colIndex = pFilterInfo->pFilter.colIdx; + SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; + int32_t colIndex = pFilterInfo->info.colIdx; // this column not valid in current data block - if (colIndex < 0 || pField[colIndex].colId != pFilterInfo->pFilter.data.colId) { + if (colIndex < 0 || pField[colIndex].colId != pFilterInfo->info.data.colId) { continue; } // not support pre-filter operation on binary/nchar data type - if (!vnodeSupportPrefilter(pFilterInfo->pFilter.data.type)) { + if (!vnodeSupportPrefilter(pFilterInfo->info.data.type)) { continue; } - if (pFilterInfo->pFilter.data.type == TSDB_DATA_TYPE_FLOAT) { + if (pFilterInfo->info.data.type == TSDB_DATA_TYPE_FLOAT) { float minval = *(double *)(&pField[colIndex].min); float maxval = *(double *)(&pField[colIndex].max); - if (!pFilterInfo->fp(&pFilterInfo->pFilter, (char *)&minval, (char *)&maxval)) { - return false; + for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) { + if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval)) { + return true; + } } } else { - if (!pFilterInfo->fp(&pFilterInfo->pFilter, (char *)&pField[colIndex].min, (char *)&pField[colIndex].max)) { - return false; + for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) { + if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pField[colIndex].min, + (char *)&pField[colIndex].max)) { + return true; + } } } } for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functId = pQuery->pSelectExpr[i].pBase.functionId; - if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_TOP_DST || functId == TSDB_FUNC_BOTTOM || - functId == TSDB_FUNC_BOTTOM_DST) { + if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { return top_bot_datablock_filter(&pCtx[i], functId, (char *)&pField[i].min, (char *)&pField[i].max); } } @@ -1175,20 +1220,189 @@ static bool needToLoadDataBlock(SQuery *pQuery, SField *pField, SQLFunctionCtx * return true; } -static int32_t applyAllFunctions_Filter(SQueryRuntimeEnv *pRuntimeEnv, int32_t *forwardStep, TSKEY *primaryKeyCol, +static int32_t setGroupResultForKey(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, char *columnData) { + SOutputRes *pOutputRes = NULL; + + // ignore the null value + if (isNull(pData, type)) { + return -1; + } + + int64_t t = 0; + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + t = GET_INT8_VAL(pData); + break; + case TSDB_DATA_TYPE_BIGINT: + t = GET_INT64_VAL(pData); + break; + case TSDB_DATA_TYPE_SMALLINT: + t = GET_INT16_VAL(pData); + break; + case TSDB_DATA_TYPE_INT: + default: + t = GET_INT32_VAL(pData); + break; + } + + SOutputRes **p1 = (SOutputRes **)taosGetIntHashData(pRuntimeEnv->hashList, t); + if (p1 != NULL) { + pOutputRes = *p1; + } else { + // more than the threshold number, discard data that are not belong to current groups + if (pRuntimeEnv->usedIndex >= 10000) { + return -1; + } + + // add a new result set for a new group + char *b = (char *)&pRuntimeEnv->pResult[pRuntimeEnv->usedIndex++]; + pOutputRes = *(SOutputRes **)taosAddIntHash(pRuntimeEnv->hashList, t, (char *)&b); + } + + setGroupOutputBuffer(pRuntimeEnv, pOutputRes); + initCtxOutputBuf(pRuntimeEnv); + + return TSDB_CODE_SUCCESS; +} + +static char *getGroupbyColumnData(SQueryRuntimeEnv *pRuntimeEnv, SField *pFields, SBlockInfo *pBlockInfo, char *data, + bool isDiskFileBlock, int16_t *type, int16_t *bytes) { + SQuery *pQuery = pRuntimeEnv->pQuery; + char * groupbyColumnData = NULL; + + int32_t col = 0; + int16_t colIndexInBuf = 0; + + SSqlGroupbyExpr *pGroupbyExpr = pQuery->pGroupbyExpr; + + for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { + if (pGroupbyExpr->columnInfo[k].flag == TSDB_COL_TAG) { + continue; + } + + int32_t colId = pGroupbyExpr->columnInfo[k].colId; + + if (isDiskFileBlock) { // get the required column data in file block according the column ID + for (int32_t i = 0; i < pBlockInfo->numOfCols; ++i) { + if (colId == pFields[i].colId) { + *type = pFields[i].type; + *bytes = pFields[i].bytes; + col = i; + break; + } + } + + // this column may not in current data block and also not in the required columns list + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + if (colId == pQuery->colList[i].data.colId) { + colIndexInBuf = i; + break; + } + } + } else { // get the required data column in cache + SColumn *pSchema = pRuntimeEnv->pMeterObj->schema; + + for (int32_t i = 0; i < pRuntimeEnv->pMeterObj->numOfColumns; ++i) { + if (colId == pSchema[i].colId) { + *type = pSchema[i].type; + *bytes = pSchema[i].bytes; + + col = i; + colIndexInBuf = i; + break; + } + } + } + + int32_t columnIndex = isDiskFileBlock ? colIndexInBuf : col; + groupbyColumnData = + doGetDataBlocks(isDiskFileBlock, pRuntimeEnv, data, columnIndex, colId, *type, *bytes, colIndexInBuf); + + break; + } + + return groupbyColumnData; +} + +static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf); + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; + + // compare tag first + if (pCtx[0].tag.i64Key != elem.tag) { + return TS_JOIN_TAG_NOT_EQUALS; + } + + TSKEY key = *(TSKEY *)(pCtx[0].aInputElemBuf + TSDB_KEYSIZE * offset); + +#if 1 + printf("elem in comp ts file:%lld, key:%lld, tag:%d, id:%s, query order:%d, ts order:%d, traverse:%d, index:%d\n", + elem.ts, key, elem.tag, pRuntimeEnv->pMeterObj->meterId, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder, + pRuntimeEnv->pTSBuf->cur.order, pRuntimeEnv->pTSBuf->cur.tsIndex); +#endif + + if (QUERY_IS_ASC_QUERY(pQuery)) { + if (key < elem.ts) { + return TS_JOIN_TS_NOT_EQUALS; + } else if (key > elem.ts) { + assert(false); + } + } else { + if (key > elem.ts) { + return TS_JOIN_TS_NOT_EQUALS; + } else if (key < elem.ts) { + assert(false); + } + } + + return TS_JOIN_TS_EQUAL; +} + +static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + + if (pResInfo->complete || functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { + return false; + } + + if (!IS_MASTER_SCAN(pRuntimeEnv) && + !(functionId == TSDB_FUNC_LAST_DST || functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_FIRST || + functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS)) { + return false; + } + + return true; +} + +static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t *forwardStep, TSKEY *primaryKeyCol, char *data, SField *pFields, SBlockInfo *pBlockInfo, bool isDiskFileBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; SQuery * pQuery = pRuntimeEnv->pQuery; - int64_t prevNumOfRes = getNumOfResult(pRuntimeEnv); + int64_t prevNumOfRes = 0; + bool groupbyStateValue = isGroupbyNormalCol(pQuery->pGroupbyExpr); + + if (!groupbyStateValue) { + prevNumOfRes = getNumOfResult(pRuntimeEnv); + } + + SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); - SArithmeticSupport *sasArray = calloc(pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); + int16_t type = 0; + int16_t bytes = 0; + + char *groupbyColumnData = NULL; + if (groupbyStateValue) { + groupbyColumnData = getGroupbyColumnData(pRuntimeEnv, pFields, pBlockInfo, data, isDiskFileBlock, &type, &bytes); + } for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; bool hasNull = hasNullVal(pQuery, k, pBlockInfo, pFields, isDiskFileBlock); - char *dataBlock = getDataBlocks(pRuntimeEnv, data, &sasArray[k], k, isDiskFileBlock); + char *dataBlock = getDataBlocks(pRuntimeEnv, data, &sasArray[k], k, *forwardStep, isDiskFileBlock); TSKEY ts = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->skey : pQuery->ekey; int64_t alignedTimestamp = taosGetIntervalStartTimestamp(ts, pQuery->nAggTimeInterval, pQuery->intervalTimeUnit, @@ -1200,12 +1414,16 @@ static int32_t applyAllFunctions_Filter(SQueryRuntimeEnv *pRuntimeEnv, int32_t * // set the input column data for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { - SColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - int32_t colIdx = isDiskFileBlock ? pFilterInfo->pFilter.colIdxInBuf : pFilterInfo->pFilter.colIdx; - SColumnFilterMsg * pFilterMsg = &pFilterInfo->pFilter.data; - /* NOTE: here the tbname/tags column cannot reach here, so we do NOT check if is a tag or not */ - pFilterInfo->pData = doGetDataBlocks(isDiskFileBlock, pRuntimeEnv, data, colIdx, pFilterMsg->colId, - pFilterMsg->type, pFilterMsg->bytes, pFilterInfo->pFilter.colIdxInBuf); + SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; + int32_t colIdx = isDiskFileBlock ? pFilterInfo->info.colIdxInBuf : pFilterInfo->info.colIdx; + SColumnInfo * pColumnInfo = &pFilterInfo->info.data; + + /* + * NOTE: here the tbname/tags column cannot reach here, since it will never be a filter column, + * so we do NOT check if is a tag or not + */ + pFilterInfo->pData = doGetDataBlocks(isDiskFileBlock, pRuntimeEnv, data, colIdx, pColumnInfo->colId, + pColumnInfo->type, pColumnInfo->bytes, pFilterInfo->info.colIdxInBuf); } int32_t numOfRes = 0; @@ -1213,23 +1431,62 @@ static int32_t applyAllFunctions_Filter(SQueryRuntimeEnv *pRuntimeEnv, int32_t * // from top to bottom in desc // from bottom to top in asc order + if (pRuntimeEnv->pTSBuf != NULL) { + SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery); + qTrace("QInfo:%p process data rows, numOfRows:%d, query order:%d, ts comp order:%d", pQInfo, *forwardStep, + pQuery->order.order, pRuntimeEnv->pTSBuf->cur.order); + } + for (int32_t j = 0; j < (*forwardStep); ++j) { - if (!vnodeDoFilterData(pQuery, pQuery->pos + j * step)) { + int32_t offset = GET_COL_DATA_POS(pQuery, j, step); + + if (pRuntimeEnv->pTSBuf != NULL) { + int32_t r = doTSJoinFilter(pRuntimeEnv, offset); + + if (r == TS_JOIN_TAG_NOT_EQUALS) { + break; + } else if (r == TS_JOIN_TS_NOT_EQUALS) { + continue; + } else { + assert(r == TS_JOIN_TS_EQUAL); + } + } + + if (pQuery->numOfFilterCols > 0 && (!vnodeDoFilterData(pQuery, offset))) { continue; } + // decide which group this rows belongs to according to current state value + if (groupbyStateValue) { + char *stateVal = groupbyColumnData + bytes * offset; + + int32_t ret = setGroupResultForKey(pRuntimeEnv, stateVal, type, groupbyColumnData); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + continue; + } + } + + // all startOffset are identical + offset -= pCtx[0].startOffset; + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { - if (pRuntimeEnv->go[k]) { - int32_t offset = pQuery->pos + j * step; - offset -= pCtx[k].startOffset; + int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunctionF(&pCtx[k], offset); + } + } - pRuntimeEnv->go[k] = aAggs[pQuery->pSelectExpr[k].pBase.functionId].xFunctionF(&pCtx[k], offset); + if (pRuntimeEnv->pTSBuf != NULL) { + // if timestamp filter list is empty, quit current query + if (!tsBufNextPos(pRuntimeEnv->pTSBuf)) { + setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + break; } } /* - * pointsOffset is the maximum available space in result buffer - * update the actual forward step for query that requires checking buffer during loop + * pointsOffset is the maximum available space in result buffer update the actual forward step for query that + * requires checking buffer during loop */ if ((pQuery->checkBufferInLoop == 1) && (++numOfRes) >= pQuery->pointsOffset) { pQuery->lastKey = primaryKeyCol[pQuery->pos + j * step] + step; @@ -1240,7 +1497,16 @@ static int32_t applyAllFunctions_Filter(SQueryRuntimeEnv *pRuntimeEnv, int32_t * free(sasArray); - return getNumOfResult(pRuntimeEnv) - prevNumOfRes; + /* + * No need to calculate the number of output results for groupby normal columns + * because the results of group by normal column is put into intermediate buffer. + */ + int32_t num = 0; + if (!groupbyStateValue) { + num = getNumOfResult(pRuntimeEnv) - prevNumOfRes; + } + + return num; } static int32_t getForwardStepsInBlock(int32_t numOfPoints, __block_search_fn_t searchFn, SQuery *pQuery, @@ -1252,8 +1518,7 @@ static int32_t getForwardStepsInBlock(int32_t numOfPoints, __block_search_fn_t s forwardStep = QUERY_IS_ASC_QUERY(pQuery) ? (endPos - pQuery->pos) : (pQuery->pos - endPos); assert(forwardStep >= 0); - /* endPos data is equalled to the key so, we do need to read the element in - * endPos */ + // endPos data is equalled to the key so, we do need to read the element in endPos if (pData[endPos] == pQuery->ekey) { forwardStep += 1; } @@ -1261,19 +1526,26 @@ static int32_t getForwardStepsInBlock(int32_t numOfPoints, __block_search_fn_t s return forwardStep; } -static int32_t reviseForwardSteps(SQuery *pQuery, int32_t forwardStep) { +static int32_t reviseForwardSteps(SQueryRuntimeEnv *pRuntimeEnv, int32_t forwardStep) { /* - * if value filter exists, we try all data in current block, and do not set the QUERY_RESBUF_FULL flag. + * 1. If value filter exists, we try all data in current block, and do not set the QUERY_RESBUF_FULL flag. + * + * 2. In case of top/bottom/ts_comp query, the checkBufferInLoop == 1 and pQuery->numOfFilterCols + * may be 0 or not. We do not check the capacity of output buffer, since the filter function will do it. * - * in handing of top/bottom query, the checkBufferInLoop == 1 and pQuery->numOfFilterCols may be 0 or not, - * so we have to exclude the query of top/bottom from checking for buffer status. + * 3. In handling the query of secondary query of join, tsBuf servers as a ts filter. */ - if (pQuery->checkBufferInLoop == 1 && pQuery->numOfFilterCols == 0 && !isTopBottomQuery(pQuery)) { - /* current buffer does not have enough space, try in the next loop */ - if (pQuery->pointsOffset <= forwardStep) { - forwardStep = pQuery->pointsOffset; - } + SQuery *pQuery = pRuntimeEnv->pQuery; + + if (isTopBottomQuery(pQuery) || isTSCompQuery(pQuery) || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { + return forwardStep; + } + + // current buffer does not have enough space, try in the next loop + if ((pQuery->checkBufferInLoop == 1) && (pQuery->pointsOffset <= forwardStep)) { + forwardStep = pQuery->pointsOffset; } + return forwardStep; } @@ -1314,8 +1586,7 @@ static int32_t applyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SBlockInfo * assert(forwardStep >= 0); if (forwardStep == 0) { - /* no qualified data in current block, do not update the lastKey value - */ + // no qualified data in current block, do not update the lastKey value assert(pQuery->ekey < pPrimaryColumn[pQuery->pos]); } else { pQuery->lastKey = pPrimaryColumn[pQuery->pos + (forwardStep - 1)] + step; @@ -1333,8 +1604,7 @@ static int32_t applyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SBlockInfo * assert(forwardStep >= 0); if (forwardStep == 0) { - /* no qualified data in current block, do not update the lastKey value - */ + // no qualified data in current block, do not update the lastKey value assert(pQuery->ekey > pPrimaryColumn[pQuery->pos]); } else { pQuery->lastKey = pPrimaryColumn[pQuery->pos - (forwardStep - 1)] + step; @@ -1347,20 +1617,22 @@ static int32_t applyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SBlockInfo * } } - int32_t newForwardStep = reviseForwardSteps(pQuery, forwardStep); + int32_t newForwardStep = reviseForwardSteps(pRuntimeEnv, forwardStep); assert(newForwardStep <= forwardStep && newForwardStep >= 0); - /* if buffer limitation is applied, there must be primary column(timestamp) loaded */ + // if buffer limitation is applied, there must be primary column(timestamp) loaded if (newForwardStep < forwardStep && newForwardStep > 0) { pQuery->lastKey = pPrimaryColumn[pQuery->pos + (newForwardStep - 1) * step] + step; } bool isFileBlock = IS_FILE_BLOCK(pRuntimeEnv->blockStatus); - if (pQuery->numOfFilterCols > 0) { + + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { *numOfRes = - applyAllFunctions_Filter(pRuntimeEnv, &newForwardStep, pPrimaryColumn, sdata, pFields, pBlockInfo, isFileBlock); + rowwiseApplyAllFunctions(pRuntimeEnv, &newForwardStep, pPrimaryColumn, sdata, pFields, pBlockInfo, isFileBlock); } else { - *numOfRes = applyAllFunctions(pRuntimeEnv, newForwardStep, pPrimaryColumn, sdata, pFields, pBlockInfo, isFileBlock); + *numOfRes = blockwiseApplyAllFunctions(pRuntimeEnv, newForwardStep, pPrimaryColumn, sdata, pFields, pBlockInfo, + isFileBlock); } assert(*numOfRes >= 0); @@ -1368,7 +1640,8 @@ static int32_t applyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SBlockInfo * // check if buffer is large enough for accommodating all qualified points if (*numOfRes > 0 && pQuery->checkBufferInLoop == 1) { pQuery->pointsOffset -= *numOfRes; - if (pQuery->pointsOffset == 0) { + if (pQuery->pointsOffset <= 0) { // todo return correct numOfRes for ts_comp function + pQuery->pointsOffset = 0; setQueryStatus(pQuery, QUERY_RESBUF_FULL); } } @@ -1401,7 +1674,7 @@ int32_t vnodeGetVnodeHeaderFileIdx(int32_t *fid, SQueryRuntimeEnv *pRuntimeEnv, int32_t i = 0; int32_t step = 1; - while(i < numOfFiles && *fid > pRuntimeEnv->pHeaderFiles[i].fileID) { + while (i pRuntimeEnv->pHeaderFiles[i].fileID) { i += step; } @@ -1415,7 +1688,7 @@ int32_t vnodeGetVnodeHeaderFileIdx(int32_t *fid, SQueryRuntimeEnv *pRuntimeEnv, int32_t i = numOfFiles - 1; int32_t step = -1; - while(i >= 0 && *fid < pRuntimeEnv->pHeaderFiles[i].fileID) { + while (i >= 0 && *fid < pRuntimeEnv->pHeaderFiles[i].fileID) { i += step; } @@ -1474,59 +1747,58 @@ int32_t getNextDataFileCompInfo(SQueryRuntimeEnv *pRuntimeEnv, SMeterObj *pMeter return fid; } -void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, int64_t StartQueryTimestamp, void *inputData, +void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, int64_t startQueryTimestamp, void *inputData, char *primaryColumnData, int32_t size, int32_t functionId, SField *pField, bool hasNull, int32_t blockStatus, void *param, int32_t scanFlag) { int32_t startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? pQuery->pos : pQuery->pos - (size - 1); - pCtx->nStartQueryTimestamp = StartQueryTimestamp; + pCtx->nStartQueryTimestamp = startQueryTimestamp; pCtx->scanFlag = scanFlag; pCtx->aInputElemBuf = inputData; - pCtx->hasNullValue = hasNull; + pCtx->hasNull = hasNull; pCtx->blockStatus = blockStatus; if (pField != NULL) { pCtx->preAggVals.isSet = true; - pCtx->preAggVals.wsum = pField->wsum; + pCtx->preAggVals.minIndex = pField->minIndex; + pCtx->preAggVals.maxIndex = pField->maxIndex; pCtx->preAggVals.sum = pField->sum; pCtx->preAggVals.max = pField->max; pCtx->preAggVals.min = pField->min; - pCtx->preAggVals.numOfNullPoints = pField->numOfNullPoints; + pCtx->preAggVals.numOfNull = pField->numOfNullPoints; } else { pCtx->preAggVals.isSet = false; } + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0 && (primaryColumnData != NULL)) { + pCtx->ptsList = (int64_t *)(primaryColumnData + startOffset * TSDB_KEYSIZE); + } + if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { // last_dist or first_dist function // store the first&last timestamp into the intermediate buffer [1], the true // value may be null but timestamp will never be null pCtx->ptsList = (int64_t *)(primaryColumnData + startOffset * TSDB_KEYSIZE); - } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_WAVG || - functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_TOP_DST || functionId == TSDB_FUNC_BOTTOM_DST) { + } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA || + functionId == TSDB_FUNC_DIFF) { /* - * leastsquares function needs two columns of input, - * currently, the x value of linear equation is set to timestamp column, and the y-value is the column specified in - * pQuery->pSelectExpr[i].colIdxInBuffer + * leastsquares function needs two columns of input, currently, the x value of linear equation is set to + * timestamp column, and the y-value is the column specified in pQuery->pSelectExpr[i].colIdxInBuffer * * top/bottom function needs timestamp to indicate when the * top/bottom values emerge, so does diff function - * */ - if (functionId == TSDB_FUNC_WAVG) { - pCtx->intermediateBuf[3].nType = TSDB_DATA_TYPE_TIMESTAMP; - pCtx->intermediateBuf[3].i64Key = pQuery->ekey; + if (functionId == TSDB_FUNC_TWA) { + STwaInfo *pTWAInfo = GET_RES_INFO(pCtx)->interResultBuf; + pTWAInfo->SKey = pQuery->skey; + pTWAInfo->EKey = pQuery->ekey; } pCtx->ptsList = (int64_t *)(primaryColumnData + startOffset * TSDB_KEYSIZE); } else if (functionId == TSDB_FUNC_ARITHM) { pCtx->param[0].pz = param; - } else if (functionId == TSDB_FUNC_WAVG_DST) { - ((SWavgRuntime *)pCtx->aOutputBuf)->sKey = StartQueryTimestamp; - ((SWavgRuntime *)pCtx->aOutputBuf)->eKey = pQuery->ekey; - - pCtx->ptsList = (int64_t *)(primaryColumnData + startOffset * TSDB_KEYSIZE); } pCtx->startOffset = startOffset; @@ -1548,23 +1820,50 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, int64_t StartQueryTimes #endif } +// set the output buffer for the selectivity + tag query +static void setCtxTagColumnInfo(SQuery* pQuery, SQueryRuntimeEnv* pRuntimeEnv) { + if (isSelectivityWithTagsQuery(pQuery)) { + int32_t num = 0; + SQLFunctionCtx *pCtx = NULL; + int16_t tagLen = 0; + + SQLFunctionCtx ** pTagCtx = calloc(pQuery->numOfOutputCols, POINTER_BYTES); + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SSqlFuncExprMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; + if (pSqlFuncMsg->functionId == TSDB_FUNC_TAG_DUMMY || pSqlFuncMsg->functionId == TSDB_FUNC_TS_DUMMY) { + tagLen += pRuntimeEnv->pCtx[i].outputBytes; + pTagCtx[num++] = &pRuntimeEnv->pCtx[i]; + } else if ((aAggs[pSqlFuncMsg->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + pCtx = &pRuntimeEnv->pCtx[i]; + } else if (pSqlFuncMsg->functionId == TSDB_FUNC_TS || pSqlFuncMsg->functionId == TSDB_FUNC_TAG) { + // tag function may be the group by tag column + // ts may be the required primary timestamp column + continue; + } else { + assert(0); + } + } + + pCtx->tagInfo.pTagCtxList = pTagCtx; + pCtx->tagInfo.numOfTagCols = num; + pCtx->tagInfo.tagsLen = tagLen; + } +} + static int32_t setupQueryRuntimeEnv(SMeterObj *pMeterObj, SQuery *pQuery, SQueryRuntimeEnv *pRuntimeEnv, - SSchema *pTagsSchema, int16_t order) { + SSchema *pTagsSchema, int16_t order, bool isMetricQuery) { dTrace("QInfo:%p setup runtime env", GET_QINFO_ADDR(pQuery)); pRuntimeEnv->pMeterObj = pMeterObj; pRuntimeEnv->pQuery = pQuery; - //todo free all allocated resource - pRuntimeEnv->go = (bool *)malloc(sizeof(bool) * pQuery->numOfOutputCols); - if (pRuntimeEnv->go == NULL) { - return TSDB_CODE_SERV_OUT_OF_MEMORY; - } - - memset(pRuntimeEnv->go, 1, sizeof(bool) * pQuery->numOfOutputCols); - + pRuntimeEnv->resultInfo = calloc(pQuery->numOfOutputCols, sizeof(SResultInfo)); pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutputCols, sizeof(SQLFunctionCtx)); + if (pRuntimeEnv->resultInfo == NULL || pRuntimeEnv->pCtx == NULL) { + goto _error_clean; + } + pRuntimeEnv->offset[0] = 0; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { SSqlFuncExprMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; @@ -1572,7 +1871,7 @@ static int32_t setupQueryRuntimeEnv(SMeterObj *pMeterObj, SQuery *pQuery, SQuery SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - if (pSqlFuncMsg->colInfo.isTag) { // process tag column info + if (TSDB_COL_IS_TAG(pSqlFuncMsg->colInfo.flag)) { // process tag column info pCtx->inputType = pTagsSchema[pColIndexEx->colIdx].type; pCtx->inputBytes = pTagsSchema[pColIndexEx->colIdx].bytes; } else { @@ -1584,23 +1883,26 @@ static int32_t setupQueryRuntimeEnv(SMeterObj *pMeterObj, SQuery *pQuery, SQuery pCtx->outputBytes = pQuery->pSelectExpr[i].resBytes; pCtx->outputType = pQuery->pSelectExpr[i].resType; + pCtx->order = pQuery->order.order; + pCtx->functionId = pSqlFuncMsg->functionId; + /* + * tricky: in case of char array parameters, we employ the shallow copy + * method and get the ownership of the char array, it later release the allocated memory if exists + */ pCtx->numOfParams = pSqlFuncMsg->numOfParams; for (int32_t j = 0; j < pCtx->numOfParams; ++j) { - /* - * tricky: in case of char array parameters, we employ the shallow copy method - * and get the ownership of the char array, it later release the allocated memory if exists - */ pCtx->param[j].nType = pSqlFuncMsg->arg[j].argType; pCtx->param[j].i64Key = pSqlFuncMsg->arg[j].argValue.i64; } - /* set the order information for top/bottom query */ - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + // set the order information for top/bottom query + int32_t functionId = pCtx->functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - assert(pQuery->pSelectExpr[0].pBase.functionId == TSDB_FUNC_TS || - pQuery->pSelectExpr[0].pBase.functionId == TSDB_FUNC_TS_DUMMY); + int32_t f = pQuery->pSelectExpr[0].pBase.functionId; + assert(f == TSDB_FUNC_TS || f == TSDB_FUNC_TS_DUMMY); pCtx->param[2].i64Key = order; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; @@ -1613,29 +1915,38 @@ static int32_t setupQueryRuntimeEnv(SMeterObj *pMeterObj, SQuery *pQuery, SQuery if (i > 0) { pRuntimeEnv->offset[i] = pRuntimeEnv->offset[i - 1] + pRuntimeEnv->pCtx[i - 1].outputBytes; } + + // set the intermediate result output buffer + SResultInfo *pResInfo = &pRuntimeEnv->resultInfo[i]; + setResultInfoBuf(pResInfo, pQuery->pSelectExpr[i].interResBytes, isMetricQuery); } - resetCtxOutputBuf(pRuntimeEnv); - initCtxOutputBuf(pRuntimeEnv); + // if it is group by normal column, do not set output buffer, the output buffer is pResult + if (!isGroupbyNormalCol(pQuery->pGroupbyExpr) && !isMetricQuery) { + resetCtxOutputBuf(pRuntimeEnv); + } - /* for loading block data in memory */ + setCtxTagColumnInfo(pQuery, pRuntimeEnv); + + // for loading block data in memory assert(vnodeList[pMeterObj->vnode].cfg.rowsInFileBlock == pMeterObj->pointsPerFileBlock); - pRuntimeEnv->buffer = - (char *)malloc(pQuery->dataRowSize * pMeterObj->pointsPerFileBlock + sizeof(SData) * pQuery->numOfCols); + size_t loadDataBlockBufferSize = + pQuery->dataRowSize * (pMeterObj->pointsPerFileBlock) + (sizeof(SData) + EXTRA_BYTES) * pQuery->numOfCols; + pRuntimeEnv->buffer = (char *)malloc(loadDataBlockBufferSize); if (pRuntimeEnv->buffer == NULL) { - return TSDB_CODE_SERV_OUT_OF_MEMORY; + goto _error_clean; } pRuntimeEnv->colDataBuffer[0] = (SData *)pRuntimeEnv->buffer; for (int32_t i = 1; i < pQuery->numOfCols; ++i) { int32_t bytes = pQuery->colList[i - 1].data.bytes; - pRuntimeEnv->colDataBuffer[i] = - (SData *)(((void *)pRuntimeEnv->colDataBuffer[i - 1]) + sizeof(SData) + pMeterObj->pointsPerFileBlock * bytes); + pRuntimeEnv->colDataBuffer[i] = (SData *)(((void *)pRuntimeEnv->colDataBuffer[i - 1]) + sizeof(SData) + + EXTRA_BYTES + pMeterObj->pointsPerFileBlock * bytes); } - /* record the maximum column width among columns of this meter/metric */ + // record the maximum column width among columns of this meter/metric int32_t maxColWidth = pQuery->colList[0].data.bytes; for (int32_t i = 1; i < pQuery->numOfCols; ++i) { int32_t bytes = pQuery->colList[i].data.bytes; @@ -1648,27 +1959,37 @@ static int32_t setupQueryRuntimeEnv(SMeterObj *pMeterObj, SQuery *pQuery, SQuery if (PRIMARY_TSCOL_LOADED(pQuery)) { pRuntimeEnv->primaryColBuffer = pRuntimeEnv->colDataBuffer[0]; } else { - pRuntimeEnv->primaryColBuffer = (SData *)malloc(pMeterObj->pointsPerFileBlock * TSDB_KEYSIZE + sizeof(SData)); + pRuntimeEnv->primaryColBuffer = + (SData *)malloc(pMeterObj->pointsPerFileBlock * TSDB_KEYSIZE + sizeof(SData) + EXTRA_BYTES); } - pRuntimeEnv->internalBufSize = (size_t) (maxColWidth*pMeterObj->pointsPerFileBlock + EXTRA_BYTES);//plus extra_bytes + pRuntimeEnv->unzipBufSize = (size_t)(maxColWidth * pMeterObj->pointsPerFileBlock + EXTRA_BYTES); // plus extra_bytes - pRuntimeEnv->unzipBuffer = (char *)malloc(pRuntimeEnv->internalBufSize); - pRuntimeEnv->secondaryUnzipBuffer = (char *)calloc(1, pRuntimeEnv->internalBufSize); + pRuntimeEnv->unzipBuffer = (char *)malloc(pRuntimeEnv->unzipBufSize); + pRuntimeEnv->secondaryUnzipBuffer = (char *)calloc(1, pRuntimeEnv->unzipBufSize); - if (pRuntimeEnv->unzipBuffer == NULL) { - return TSDB_CODE_SERV_OUT_OF_MEMORY; + if (pRuntimeEnv->unzipBuffer == NULL || pRuntimeEnv->secondaryUnzipBuffer == NULL || + pRuntimeEnv->primaryColBuffer == NULL) { + goto _error_clean; } - if (pRuntimeEnv->secondaryUnzipBuffer == NULL) { - return TSDB_CODE_SERV_OUT_OF_MEMORY; + return TSDB_CODE_SUCCESS; + +_error_clean: + tfree(pRuntimeEnv->resultInfo); + tfree(pRuntimeEnv->pCtx); + tfree(pRuntimeEnv->buffer); + tfree(pRuntimeEnv->unzipBuffer); + tfree(pRuntimeEnv->secondaryUnzipBuffer); + + if (!PRIMARY_TSCOL_LOADED(pQuery)) { + tfree(pRuntimeEnv->primaryColBuffer); } - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SERV_OUT_OF_MEMORY; } static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { - /* query runtime env is not called */ if (pRuntimeEnv->pQuery == NULL) { return; } @@ -1676,7 +1997,8 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { dTrace("QInfo:%p teardown runtime env", GET_QINFO_ADDR(pRuntimeEnv->pQuery)); tfree(pRuntimeEnv->buffer); tfree(pRuntimeEnv->secondaryUnzipBuffer); - tfree(pRuntimeEnv->go); + + taosCleanUpIntHash(pRuntimeEnv->hashList); if (pRuntimeEnv->pCtx != NULL) { for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutputCols; ++i) { @@ -1686,11 +2008,11 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { tVariantDestroy(&pCtx->param[j]); } - for (int32_t j = 0; j < tListLen(pCtx->intermediateBuf); ++j) { - tVariantDestroy(&pCtx->intermediateBuf[j]); - } + tVariantDestroy(&pCtx->tag); + tfree(pRuntimeEnv->resultInfo[i].interResultBuf); } + tfree(pRuntimeEnv->resultInfo); tfree(pRuntimeEnv->pCtx); } @@ -1727,6 +2049,11 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { tfree(pRuntimeEnv->pInterpoBuf); } + + if (pRuntimeEnv->pTSBuf != NULL) { + tsBufDestory(pRuntimeEnv->pTSBuf); + pRuntimeEnv->pTSBuf = NULL; + } } // get maximum time interval in each file @@ -1742,7 +2069,7 @@ bool isQueryKilled(SQuery *pQuery) { * check if the queried meter is going to be deleted. * if it will be deleted soon, stop current query ASAP. */ - SMeterObj* pMeterObj = pQInfo->pObj; + SMeterObj *pMeterObj = pQInfo->pObj; if (vnodeIsMeterState(pMeterObj, TSDB_METER_STATE_DELETING)) { pQInfo->killed = 1; return true; @@ -1757,24 +2084,40 @@ bool isFixedOutputQuery(SQuery *pQuery) { } // Note:top/bottom query is fixed output query - if (isTopBottomQuery(pQuery)) { + if (isTopBottomQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { return true; } for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; - if (IS_MULTIOUTPUT(aAggs[functionId].nStatus)) { - return false; + SSqlFuncExprMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; + + // ignore the ts_comp function + if (i == 0 && pExprMsg->functionId == TSDB_FUNC_PRJ && pExprMsg->numOfParams == 1 && + pExprMsg->colInfo.colIdx == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + continue; } - } - return true; -} + if (pExprMsg->functionId == TSDB_FUNC_TS || pExprMsg->functionId == TSDB_FUNC_TS_DUMMY) { + continue; + } -bool isPointInterpoQuery(SQuery *pQuery) { + // // ignore the group by + projection combination + // if (pExprMsg->functionId == TSDB_FUNC_PRJ && isGroupbyNormalCol(pQuery)) { + // continue; + // } + + if (!IS_MULTIOUTPUT(aAggs[pExprMsg->functionId].nStatus)) { + return true; + } + } + + return false; +} + +bool isPointInterpoQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionID = pQuery->pSelectExpr[i].pBase.functionId; - if (functionID == TSDB_FUNC_INTERP || functionID == TSDB_FUNC_LAST_ROW || functionID == TSDB_FUNC_LAST_ROW_DST) { + if (functionID == TSDB_FUNC_INTERP || functionID == TSDB_FUNC_LAST_ROW) { return true; } } @@ -1790,8 +2133,7 @@ bool isTopBottomQuery(SQuery *pQuery) { continue; } - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TOP_DST || - functionId == TSDB_FUNC_BOTTOM_DST) { + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { return true; } } @@ -1802,7 +2144,7 @@ bool isTopBottomQuery(SQuery *pQuery) { bool isFirstLastRowQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionID = pQuery->pSelectExpr[i].pBase.functionId; - if (functionID == TSDB_FUNC_LAST_ROW || functionID == TSDB_FUNC_LAST_ROW_DST) { + if (functionID == TSDB_FUNC_LAST_ROW) { return true; } } @@ -1810,6 +2152,8 @@ bool isFirstLastRowQuery(SQuery *pQuery) { return false; } +bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].pBase.functionId == TSDB_FUNC_TS_COMP; } + bool needSupplementaryScan(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; @@ -1928,8 +2272,8 @@ static bool cacheBoundaryCheck(SQuery *pQuery, SMeterObj *pMeterObj) { /* * The query time range is earlier than the first element or later than the last elements in cache. - * If the query window happens to overlap with the time range of disk files but not data in cache, - * the flag needs to be cleared. Otherwise, this flag will cause error in following processing. + * If the query window overlaps with the time range of disk files, the flag needs to be reset. + * Otherwise, this flag will cause error in following processing. */ if (max < keyFirst || min > keyLast) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); @@ -1976,7 +2320,7 @@ int64_t getQueryStartPositionInCache(SQueryRuntimeEnv *pRuntimeEnv, int32_t *slo pQuery->fileId = -1; vnodeFreeFieldsEx(pRuntimeEnv); - //keep in-memory cache status in local variables in case that it may be changed by write operation + // keep in-memory cache status in local variables in case that it may be changed by write operation getBasicCacheInfoSnapshot(pQuery, pMeterObj->pCache, pMeterObj->vnode); SCacheInfo *pCacheInfo = (SCacheInfo *)pMeterObj->pCache; @@ -2264,8 +2608,8 @@ static bool getNeighborPoints(SMeterQuerySupportObj *pSupporter, SMeterObj *pMet */ if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK)) { dTrace("QInfo:%p no previous data block, start fileId:%d, slot:%d, pos:%d, qrange:%lld-%lld, out of range", - GET_QINFO_ADDR(pQuery), pRuntimeEnv->startPos.fileId, pRuntimeEnv->startPos.slot, pRuntimeEnv->startPos.pos, - pQuery->skey, pQuery->ekey); + GET_QINFO_ADDR(pQuery), pRuntimeEnv->startPos.fileId, pRuntimeEnv->startPos.slot, + pRuntimeEnv->startPos.pos, pQuery->skey, pQuery->ekey); // no result, return immediately setQueryStatus(pQuery, QUERY_COMPLETED); @@ -2275,8 +2619,8 @@ static bool getNeighborPoints(SMeterQuerySupportObj *pSupporter, SMeterObj *pMet pQuery->pos = pQuery->pBlock[pQuery->slot].numOfPoints - 1; getOneRowFromDiskBlock(pRuntimeEnv, pPointInterpSupporter->pPrevPoint, pQuery->pos); - qTrace("QInfo:%p get prev data point, fileId:%d, slot:%d, pos:%d, pQuery->pos:%d", - GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->slot, pQuery->pos, pQuery->pos); + qTrace("QInfo:%p get prev data point, fileId:%d, slot:%d, pos:%d, pQuery->pos:%d", GET_QINFO_ADDR(pQuery), + pQuery->fileId, pQuery->slot, pQuery->pos, pQuery->pos); } else { pBlock = getCacheDataBlock(pMeterObj, pQuery, pQuery->slot); if (pBlock == NULL) { @@ -2285,8 +2629,8 @@ static bool getNeighborPoints(SMeterQuerySupportObj *pSupporter, SMeterObj *pMet pQuery->pos = pBlock->numOfPoints - 1; getOneRowFromCacheBlock(pRuntimeEnv, pMeterObj, pBlock, pPointInterpSupporter->pPrevPoint, pQuery->pos); - qTrace("QInfo:%p get prev data point, fileId:%d, slot:%d, pos:%d, pQuery->pos:%d", - GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->slot, pBlock->numOfPoints - 1, pQuery->pos); + qTrace("QInfo:%p get prev data point, fileId:%d, slot:%d, pos:%d, pQuery->pos:%d", GET_QINFO_ADDR(pQuery), + pQuery->fileId, pQuery->slot, pBlock->numOfPoints - 1, pQuery->pos); } } } @@ -2323,7 +2667,8 @@ static bool doGetQueryPos(TSKEY key, SMeterQuerySupportObj *pSupporter, SPointIn /** * determine the first query range, according to raw query range [skey, ekey] and group-by interval. - * the time interval for aggregating is not enforced to check its validation, the minimum interval is not less than 10ms, + * the time interval for aggregating is not enforced to check its validation, the minimum interval is not less than + * 10ms, * which is guaranteed by parser at client-side */ bool normalizedFirstQueryRange(bool dataInDisk, bool dataInCache, SMeterQuerySupportObj *pSupporter, @@ -2458,19 +2803,27 @@ int64_t loadRequiredBlockIntoMem(SQueryRuntimeEnv *pRuntimeEnv, SPositionInfo *p } static void setScanLimitationByResultBuffer(SQuery *pQuery) { - bool multiOutput = false; + if (isTopBottomQuery(pQuery)) { + pQuery->checkBufferInLoop = 0; + } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + pQuery->checkBufferInLoop = 0; + } else { + bool hasMultioutput = false; + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SSqlFuncExprMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; + if (pExprMsg->functionId == TSDB_FUNC_TS || pExprMsg->functionId == TSDB_FUNC_TS_DUMMY) { + continue; + } - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; - multiOutput = IS_MULTIOUTPUT(aAggs[functionId].nStatus) && (functionId != TSDB_FUNC_TOP) && - (functionId != TSDB_FUNC_BOTTOM) && (functionId != TSDB_FUNC_TOP_DST) && - (functionId != TSDB_FUNC_BOTTOM_DST); - if (multiOutput) { - break; + hasMultioutput = IS_MULTIOUTPUT(aAggs[pExprMsg->functionId].nStatus); + if (!hasMultioutput) { + break; + } } + + pQuery->checkBufferInLoop = hasMultioutput ? 1 : 0; } - pQuery->checkBufferInLoop = multiOutput ? 1 : 0; pQuery->pointsOffset = pQuery->pointsToRead; } @@ -2546,8 +2899,8 @@ static int32_t vnodeOpenVnodeDBFiles(SQInfo *pQInfo, SQueryFileInfo *pVnodeFiles goto _clean; } + /* even the advise failed, continue.. */ if (madvise(pVnodeFiles->pHeaderFileData, size, MADV_SEQUENTIAL) == -1) { - /* even the advise failed, continue.. */ dError("QInfo:%p failed to advise kernel the usage of header files, reason:%s", pQInfo, strerror(errno)); } @@ -2577,18 +2930,13 @@ static int32_t vnodeOpenVnodeDBFiles(SQInfo *pQInfo, SQueryFileInfo *pVnodeFiles goto _clean; } -#if 0 - size = lseek(pVnodeFiles->dataFd, 0, SEEK_END); - pVnodeFiles->dataFileSize = size; - pVnodeFiles->dtFileMappingOffset = 0; -#else if (stat(pVnodeFiles->dataFilePath, &fstat) < 0) return -1; pVnodeFiles->dataFileSize = fstat.st_size; if (stat(pVnodeFiles->lastFilePath, &fstat) < 0) return -1; pVnodeFiles->lastFileSize = fstat.st_size; -#endif +#if DEFAULT_IO_ENGINE == IO_ENGINE_MMAP /* enforce kernel to preload data when the file is mapping */ pVnodeFiles->pDataFileData = mmap(NULL, pVnodeFiles->defaultMappingSize, PROT_READ, MAP_PRIVATE | MAP_POPULATE, pVnodeFiles->dataFd, pVnodeFiles->dtFileMappingOffset); @@ -2602,6 +2950,7 @@ static int32_t vnodeOpenVnodeDBFiles(SQInfo *pQInfo, SQueryFileInfo *pVnodeFiles dError("QInfo:%p failed to advise kernel the usage of data file:%s, reason:%s", pQInfo, pVnodeFiles->dataFilePath, strerror(errno)); } +#endif return 0; @@ -2611,10 +2960,12 @@ _clean: pVnodeFiles->pHeaderFileData = NULL; } +#if DEFAULT_IO_ENGINE == IO_ENGINE_MMAP if (pVnodeFiles->pDataFileData != MAP_FAILED && pVnodeFiles->pDataFileData != NULL) { munmap(pVnodeFiles->pDataFileData, pVnodeFiles->defaultMappingSize); pVnodeFiles->pDataFileData = NULL; } +#endif tclose(pVnodeFiles->headerFd); tclose(pVnodeFiles->dataFd); @@ -2694,7 +3045,7 @@ static void vnodeOpenAllFiles(SQInfo *pQInfo, int32_t vnodeId) { qsort(pRuntimeEnv->pHeaderFiles, (size_t)pRuntimeEnv->numOfFiles, sizeof(SQueryFileInfo), file_order_comparator); } -static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SBlockInfo* pBlockInfo, void *pBlock) { +static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SBlockInfo *pBlockInfo, void *pBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; /* @@ -2703,23 +3054,22 @@ static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SBlockInfo* pBlockInf */ if ((pQuery->ekey <= pBlockInfo->keyLast && QUERY_IS_ASC_QUERY(pQuery)) || (pQuery->ekey >= pBlockInfo->keyFirst && !QUERY_IS_ASC_QUERY(pQuery))) { - // force load timestamp data blocks if (IS_DISK_DATA_BLOCK(pQuery)) { getTimestampInDiskBlock(pRuntimeEnv, 0); } // update the pQuery->limit.offset value, and pQuery->pos value - TSKEY* keys = NULL; + TSKEY *keys = NULL; if (IS_DISK_DATA_BLOCK(pQuery)) { - keys = (TSKEY *) pRuntimeEnv->primaryColBuffer->data; + keys = (TSKEY *)pRuntimeEnv->primaryColBuffer->data; } else { - keys = (TSKEY *) (((SCacheBlock *)pBlock)->offset[0]); + keys = (TSKEY *)(((SCacheBlock *)pBlock)->offset[0]); } int32_t i = 0; if (QUERY_IS_ASC_QUERY(pQuery)) { - for(i = pQuery->pos; i < pBlockInfo->size && pQuery->limit.offset > 0; ++i) { + for (i = pQuery->pos; i < pBlockInfo->size && pQuery->limit.offset > 0; ++i) { if (keys[i] <= pQuery->ekey) { pQuery->limit.offset -= 1; } else { @@ -2728,7 +3078,7 @@ static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SBlockInfo* pBlockInf } } else { - for(i = pQuery->pos; i >= 0 && pQuery->limit.offset > 0; --i) { + for (i = pQuery->pos; i >= 0 && pQuery->limit.offset > 0; --i) { if (keys[i] >= pQuery->ekey) { pQuery->limit.offset -= 1; } else { @@ -2788,8 +3138,9 @@ static bool onlyLastQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSDB static void rewriteExecOrder(SQuery *pQuery, bool metricQuery) { // in case of point-interpolation query, use asc order scan - char msg[] = "QInfo:%p scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%lld-%lld, " - "new qrange:%lld-%lld"; + char msg[] = + "QInfo:%p scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%lld-%lld, " + "new qrange:%lld-%lld"; // descending order query if (isFirstLastRowQuery(pQuery)) { @@ -2864,20 +3215,21 @@ static int32_t doSkipDataBlock(SQueryRuntimeEnv *pRuntimeEnv) { __block_search_fn_t searchFn = vnodeSearchKeyFunc[pMeterObj->searchAlgorithm]; while (1) { - int32_t ret = moveToNextBlock(pRuntimeEnv, step, searchFn, false); - UNUSED(ret); + moveToNextBlock(pRuntimeEnv, step, searchFn, false); if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK)) { break; } void *pBlock = getGenericDataBlock(pMeterObj, pQuery, pQuery->slot); + assert(pBlock != NULL); int32_t blockType = IS_DISK_DATA_BLOCK(pQuery) ? BLK_FILE_BLOCK : BLK_CACHE_BLOCK; SBlockInfo blockInfo = getBlockBasicInfo(pBlock, blockType); int32_t maxReads = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.size - pQuery->pos : pQuery->pos + 1; + assert(maxReads >= 0); - if (pQuery->limit.offset < maxReads || (pQuery->ekey <= blockInfo.keyLast && QUERY_IS_ASC_QUERY(pQuery)) || + if (pQuery->limit.offset < maxReads || (pQuery->ekey <= blockInfo.keyLast && QUERY_IS_ASC_QUERY(pQuery)) || (pQuery->ekey >= blockInfo.keyFirst && !QUERY_IS_ASC_QUERY(pQuery))) { // start position in current block updateOffsetVal(pRuntimeEnv, &blockInfo, pBlock); break; @@ -2885,6 +3237,8 @@ static int32_t doSkipDataBlock(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->limit.offset -= maxReads; pQuery->lastKey = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.keyLast : blockInfo.keyFirst; pQuery->lastKey += step; + + qTrace("QInfo:%p skip rows:%d, offset:%lld", GET_QINFO_ADDR(pQuery), maxReads, pQuery->limit.offset); } } @@ -2899,13 +3253,15 @@ void forwardQueryStartPosition(SQueryRuntimeEnv *pRuntimeEnv) { return; } - void * pBlock = getGenericDataBlock(pMeterObj, pQuery, pQuery->slot); - int32_t blockType = (IS_DISK_DATA_BLOCK(pQuery)) ? BLK_FILE_BLOCK : BLK_CACHE_BLOCK; + void *pBlock = getGenericDataBlock(pMeterObj, pQuery, pQuery->slot); + int32_t blockType = (IS_DISK_DATA_BLOCK(pQuery)) ? BLK_FILE_BLOCK : BLK_CACHE_BLOCK; SBlockInfo blockInfo = getBlockBasicInfo(pBlock, blockType); - int32_t maxReads = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.size - pQuery->pos : pQuery->pos + 1; - if (pQuery->limit.offset < maxReads || (pQuery->ekey <= blockInfo.keyLast && QUERY_IS_ASC_QUERY(pQuery)) || + // get the qualified data that can be skipped + int32_t maxReads = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.size - pQuery->pos : pQuery->pos + 1; + + if (pQuery->limit.offset < maxReads || (pQuery->ekey <= blockInfo.keyLast && QUERY_IS_ASC_QUERY(pQuery)) || (pQuery->ekey >= blockInfo.keyFirst && !QUERY_IS_ASC_QUERY(pQuery))) { // start position in current block updateOffsetVal(pRuntimeEnv, &blockInfo, pBlock); } else { @@ -2919,7 +3275,7 @@ static bool forwardQueryStartPosIfNeeded(SQInfo *pQInfo, SMeterQuerySupportObj * SQuery *pQuery = &pQInfo->query; /* if queried with value filter, do NOT forward query start position */ - if (pQuery->numOfFilterCols > 0) { + if (pQuery->numOfFilterCols > 0 || pSupporter->runtimeEnv.pTSBuf != NULL) { return true; } @@ -2935,7 +3291,7 @@ static bool forwardQueryStartPosIfNeeded(SQInfo *pQInfo, SMeterQuerySupportObj * /* * the skey may not be the aligned start time * 1. it is the value of first existed data point, therefore, the range - * between skey and ekey may be less than the interval value. + * between skey and ekey may be less than the interval value. * 2. the ekey may not be the actual end value of time interval, in case of the */ if (QUERY_IS_ASC_QUERY(pQuery)) { @@ -2949,14 +3305,14 @@ static bool forwardQueryStartPosIfNeeded(SQInfo *pQInfo, SMeterQuerySupportObj * (pQuery->skey < pSupporter->rawEKey && !QUERY_IS_ASC_QUERY(pQuery))) { setQueryStatus(pQuery, QUERY_COMPLETED); - sem_post(&pQInfo->dataReady); // hack for next read for empty return + sem_post(&pQInfo->dataReady); pQInfo->over = 1; return false; } /* - * NOTE: the end key must be set the last value, to cover all possible data. Otherwise, it may - * contain no data with only one interval time range + * NOTE: the end key must be set the last value, to cover all possible + * data. Otherwise, it may contain no data with only one interval time range */ pQuery->ekey = pSupporter->rawEKey; pQuery->lastKey = pQuery->skey; @@ -2977,7 +3333,7 @@ static bool forwardQueryStartPosIfNeeded(SQInfo *pQInfo, SMeterQuerySupportObj * if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK)) { setQueryStatus(pQuery, QUERY_COMPLETED); - sem_post(&pQInfo->dataReady); // hack for next read for empty return + sem_post(&pQInfo->dataReady); // hack for next read for empty return; pQInfo->over = 1; return false; } @@ -2987,24 +3343,72 @@ static bool forwardQueryStartPosIfNeeded(SQInfo *pQInfo, SMeterQuerySupportObj * return true; } +static void doSetInterpVal(SQLFunctionCtx *pCtx, TSKEY ts, int16_t type, int32_t index, char *data) { + assert(pCtx->param[index].pz == NULL); + + int32_t len = 0; + size_t t = 0; + + if (type == TSDB_DATA_TYPE_BINARY) { + t = strlen(data); + + len = t + 1 + TSDB_KEYSIZE; + pCtx->param[index].pz = calloc(1, len); + } else if (type == TSDB_DATA_TYPE_NCHAR) { + t = wcslen((const wchar_t *)data); + + len = (t + 1) * TSDB_NCHAR_SIZE + TSDB_KEYSIZE; + pCtx->param[index].pz = calloc(1, len); + } else { + len = TSDB_KEYSIZE * 2; + pCtx->param[index].pz = malloc(len); + } + + pCtx->param[index].nType = TSDB_DATA_TYPE_BINARY; + + char *z = pCtx->param[index].pz; + *(TSKEY *)z = ts; + z += TSDB_KEYSIZE; + + switch (type) { + case TSDB_DATA_TYPE_FLOAT: + *(double *)z = GET_FLOAT_VAL(data); + break; + case TSDB_DATA_TYPE_DOUBLE: + *(double *)z = GET_DOUBLE_VAL(data); + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_TIMESTAMP: + *(int64_t *)z = GET_INT64_VAL(data); + break; + case TSDB_DATA_TYPE_BINARY: + strncpy(z, data, t); + break; + case TSDB_DATA_TYPE_NCHAR: { + wcsncpy((wchar_t *)z, (const wchar_t *)data, t); + } break; + default: + assert(0); + } + + pCtx->param[index].nLen = len; +} + /** - * param[0]: tags. reserved for tags * param[1]: default value/previous value of specified timestamp * param[2]: next value of specified timestamp * param[3]: denotes if the result is a precious result or interpolation results * - * intermediate[0]: interpolation type - * intermediate[1]: precious specified timestamp, the pCtx->startTimetamp is - * changed during query to satisfy the query procedure - * intermediate[2]: flag that denotes if it is a primary timestamp column or not - * - * todo refactor * @param pQInfo * @param pSupporter * @param pInterpoRaw */ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointInterpSupport) { - /* not point interpolation query, abort */ + // not point interpolation query, abort if (!isPointInterpoQuery(&pQInfo->query)) { return; } @@ -3017,114 +3421,74 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI TSKEY key = *(TSKEY *)pPointInterpSupport->pNextPoint[0]; if (key == pSupporter->rawSKey) { - /* the queried timestamp has value, return it directly without interpolation - */ + // the queried timestamp has value, return it directly without interpolation for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - tVariantCreateB(&pRuntimeEnv->pCtx[i].param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT); + tVariantCreateFromBinary(&pRuntimeEnv->pCtx[i].param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT); - pRuntimeEnv->pCtx[i].intermediateBuf[1].i64Key = key; - pRuntimeEnv->pCtx[i].intermediateBuf[1].nType = TSDB_DATA_TYPE_BIGINT; + pRuntimeEnv->pCtx[i].param[0].i64Key = key; + pRuntimeEnv->pCtx[i].param[0].nType = TSDB_DATA_TYPE_BIGINT; } } else { - /* set the direct previous(next) point for process */ + // set the direct previous(next) point for process count = 2; if (pQuery->interpoType == TSDB_INTERPO_SET_VALUE) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + SInterpInfo * pInterpInfo = (SInterpInfo *)pRuntimeEnv->pCtx[i].aOutputBuf; - /* for primary timestamp column, set the flag*/ - if (pQuery->pSelectExpr[i].pBase.colInfo.colId == 0) { - pCtx->intermediateBuf[2].i64Key = 1; - pCtx->intermediateBuf[2].nType = TSDB_DATA_TYPE_BIGINT; + pInterpInfo->pInterpDetail = calloc(1, sizeof(SInterpInfoDetail)); + SInterpInfoDetail *pInterpDetail = pInterpInfo->pInterpDetail; + + // for primary timestamp column, set the flag + if (pQuery->pSelectExpr[i].pBase.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + pInterpDetail->primaryCol = 1; } - tVariantCreateB(&pCtx->param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT); - tVariantCreateB(&pCtx->param[1], (char *)&pQuery->defaultVal[i], pCtx->inputBytes, pCtx->inputType); - pCtx->intermediateBuf[1].i64Key = pSupporter->rawSKey; - pCtx->intermediateBuf[1].nType = TSDB_DATA_TYPE_BIGINT; + tVariantCreateFromBinary(&pCtx->param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT); + + if (isNull((char *)&pQuery->defaultVal[i], pCtx->inputType)) { + pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; + } else { + tVariantCreateFromBinary(&pCtx->param[1], (char *)&pQuery->defaultVal[i], pCtx->inputBytes, pCtx->inputType); + } - /* the interpolation type is set in intermediateBuf[0] */ - tVariantCreateB(&pCtx->intermediateBuf[0], (char *)&pQuery->interpoType, sizeof(pQuery->interpoType), - TSDB_DATA_TYPE_SMALLINT); + pInterpDetail->ts = pSupporter->rawSKey; + pInterpDetail->type = pQuery->interpoType; } } else { TSKEY prevKey = *(TSKEY *)pPointInterpSupport->pPrevPoint[0]; TSKEY nextKey = *(TSKEY *)pPointInterpSupport->pNextPoint[0]; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + + // tag column does not need the interp environment + if (pQuery->pSelectExpr[i].pBase.functionId == TSDB_FUNC_TAG) { + continue; + } + int32_t colInBuf = pQuery->pSelectExpr[i].pBase.colInfo.colIdxInBuf; - int32_t type = GET_COLUMN_TYPE(pQuery, i); - char tmp[TSDB_MAX_BYTES_PER_ROW] = {0}; - int32_t len = 0; + SInterpInfo *pInterpInfo = (SInterpInfo *)pRuntimeEnv->pCtx[i].aOutputBuf; + + pInterpInfo->pInterpDetail = calloc(1, sizeof(SInterpInfoDetail)); + SInterpInfoDetail *pInterpDetail = pInterpInfo->pInterpDetail; - /* for primary timestamp column, set the flag*/ - if (pQuery->pSelectExpr[i].pBase.colInfo.colId == 0) { - pRuntimeEnv->pCtx[i].intermediateBuf[2].i64Key = 1; - pRuntimeEnv->pCtx[i].intermediateBuf[2].nType = TSDB_DATA_TYPE_BIGINT; + int32_t type = GET_COLUMN_TYPE(pQuery, i); + + // for primary timestamp column, set the flag + if (pQuery->pSelectExpr[i].pBase.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + pInterpDetail->primaryCol = 1; } else { - if ((type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_BIGINT) || type == TSDB_DATA_TYPE_TIMESTAMP) { - len = sprintf(tmp, "%ld,%ld", prevKey, *(int64_t *)pPointInterpSupport->pPrevPoint[colInBuf]); - tVariantCreateB(&pRuntimeEnv->pCtx[i].param[1], tmp, len, TSDB_DATA_TYPE_BINARY); - - len = sprintf(tmp, "%ld,%ld", nextKey, *(int64_t *)pPointInterpSupport->pNextPoint[colInBuf]); - tVariantCreateB(&pRuntimeEnv->pCtx[i].param[2], tmp, len, TSDB_DATA_TYPE_BINARY); - } else if (type == TSDB_DATA_TYPE_FLOAT) { - if (isNull(pPointInterpSupport->pPrevPoint[colInBuf], type)) { - tVariantCreateB(&pRuntimeEnv->pCtx[i].param[1], TSDB_DATA_NULL_STR_L, strlen(TSDB_DATA_NULL_STR_L), - TSDB_DATA_TYPE_BINARY); - } else { - len = sprintf(tmp, "%ld,%.9f", prevKey, *(float *)pPointInterpSupport->pPrevPoint[colInBuf]); - tVariantCreateB(&pRuntimeEnv->pCtx[i].param[1], tmp, len, TSDB_DATA_TYPE_BINARY); - } - - if (isNull(pPointInterpSupport->pNextPoint[colInBuf], type)) { - tVariantCreateB(&pRuntimeEnv->pCtx[i].param[1], TSDB_DATA_NULL_STR_L, strlen(TSDB_DATA_NULL_STR_L), - TSDB_DATA_TYPE_BINARY); - } else { - len = sprintf(tmp, "%ld,%.9f", nextKey, *(float *)pPointInterpSupport->pNextPoint[colInBuf]); - tVariantCreateB(&pRuntimeEnv->pCtx[i].param[2], tmp, len, TSDB_DATA_TYPE_BINARY); - } - } else if (type == TSDB_DATA_TYPE_DOUBLE) { - if (isNull(pPointInterpSupport->pPrevPoint[colInBuf], type)) { - tVariantCreateB(&pRuntimeEnv->pCtx[i].param[1], TSDB_DATA_NULL_STR_L, strlen(TSDB_DATA_NULL_STR_L), - TSDB_DATA_TYPE_BINARY); - } else { - len = sprintf(tmp, "%ld,%.9f", prevKey, *(double *)pPointInterpSupport->pPrevPoint[colInBuf]); - tVariantCreateB(&pRuntimeEnv->pCtx[i].param[1], tmp, len, TSDB_DATA_TYPE_BINARY); - } - - if (isNull(pPointInterpSupport->pPrevPoint[colInBuf], type)) { - tVariantCreateB(&pRuntimeEnv->pCtx[i].param[1], TSDB_DATA_NULL_STR_L, strlen(TSDB_DATA_NULL_STR_L), - TSDB_DATA_TYPE_BINARY); - } else { - len = sprintf(tmp, "%ld,%.9f", nextKey, *(double *)pPointInterpSupport->pNextPoint[colInBuf]); - tVariantCreateB(&pRuntimeEnv->pCtx[i].param[2], tmp, len, TSDB_DATA_TYPE_BINARY); - } - } else if (type == TSDB_DATA_TYPE_BINARY) { - len = sprintf(tmp, "%ld,%s", prevKey, pPointInterpSupport->pPrevPoint[colInBuf]); - tVariantCreateB(&pRuntimeEnv->pCtx[i].param[1], tmp, len, TSDB_DATA_TYPE_BINARY); - - len = sprintf(tmp, "%ld,%s", nextKey, pPointInterpSupport->pNextPoint[colInBuf]); - tVariantCreateB(&pRuntimeEnv->pCtx[i].param[2], tmp, len, TSDB_DATA_TYPE_BINARY); - } else if (type == TSDB_DATA_TYPE_NCHAR) { - int32_t maxLen = TSDB_MAX_BYTES_PER_ROW / TSDB_NCHAR_SIZE; - len = swprintf((wchar_t *)tmp, maxLen, L"%ld,%ls", prevKey, pPointInterpSupport->pPrevPoint[colInBuf]); - tVariantCreateB(&pRuntimeEnv->pCtx[i].param[1], tmp, len * TSDB_NCHAR_SIZE, TSDB_DATA_TYPE_NCHAR); - - len = swprintf((wchar_t *)tmp, maxLen, L"%ld,%ls", nextKey, pPointInterpSupport->pNextPoint[colInBuf]); - tVariantCreateB(&pRuntimeEnv->pCtx[i].param[2], tmp, len * TSDB_NCHAR_SIZE, TSDB_DATA_TYPE_NCHAR); - } + doSetInterpVal(pCtx, prevKey, type, 1, pPointInterpSupport->pPrevPoint[colInBuf]); + doSetInterpVal(pCtx, nextKey, type, 2, pPointInterpSupport->pNextPoint[colInBuf]); } - tVariantCreateB(&pRuntimeEnv->pCtx[i].param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT); + tVariantCreateFromBinary(&pRuntimeEnv->pCtx[i].param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT); - /* the interpolation type is set in intermediateBuf[0] */ - tVariantCreateB(&pRuntimeEnv->pCtx[i].intermediateBuf[0], (char *)&pQuery->interpoType, - sizeof(pQuery->interpoType), TSDB_DATA_TYPE_SMALLINT); - pRuntimeEnv->pCtx[i].intermediateBuf[1].i64Key = pSupporter->rawSKey; - pRuntimeEnv->pCtx[i].intermediateBuf[1].nType = TSDB_DATA_TYPE_BIGINT; + pInterpDetail->ts = pSupporter->rawSKey; + pInterpDetail->type = pQuery->interpoType; } } } @@ -3132,8 +3496,8 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI void pointInterpSupporterInit(SQuery *pQuery, SPointInterpoSupporter *pInterpoSupport) { if (isPointInterpoQuery(pQuery)) { - pInterpoSupport->pPrevPoint = malloc(pQuery->numOfCols * sizeof(void *)); - pInterpoSupport->pNextPoint = malloc(pQuery->numOfCols * sizeof(void *)); + pInterpoSupport->pPrevPoint = malloc(pQuery->numOfCols * POINTER_BYTES); + pInterpoSupport->pNextPoint = malloc(pQuery->numOfCols * POINTER_BYTES); pInterpoSupport->numOfCols = pQuery->numOfCols; @@ -3188,12 +3552,48 @@ static void allocMemForInterpo(SMeterQuerySupportObj *pSupporter, SQuery *pQuery } } -int32_t vnodeQuerySingleMeterPrepare(SQInfo *pQInfo, SMeterObj *pMeterObj, SMeterQuerySupportObj *pSupporter) { +static int32_t allocateOutputBufForGroup(SMeterQuerySupportObj *pSupporter, SQuery *pQuery, bool isMetricQuery) { + int32_t slot = 0; + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + slot = 10000; + } else { + slot = pSupporter->pSidSet->numOfSubSet; + } + + pSupporter->pResult = calloc(1, sizeof(SOutputRes) * slot); + if (pSupporter->pResult == NULL) { + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } + + // create group result buffer + for (int32_t k = 0; k < slot; ++k) { + SOutputRes *pOneRes = &pSupporter->pResult[k]; + + /* + * for top/bottom query, the output for group by normal column, the output rows is equals to the + * maximum rows, instead of 1. + */ + SSqlFunctionExpr *pExpr = &pQuery->pSelectExpr[1]; + if ((pExpr->pBase.functionId == TSDB_FUNC_TOP || pExpr->pBase.functionId == TSDB_FUNC_BOTTOM) && + pExpr->resType != TSDB_DATA_TYPE_BINARY) { + pOneRes->nAlloc = pExpr->pBase.arg[0].argValue.i64; + } else { + pOneRes->nAlloc = 1; + } + + createGroupResultBuf(pQuery, pOneRes, isMetricQuery); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t vnodeQuerySingleMeterPrepare(SQInfo *pQInfo, SMeterObj *pMeterObj, SMeterQuerySupportObj *pSupporter, + void *param) { SQuery *pQuery = &pQInfo->query; if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->skey > pQuery->ekey)) || (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->ekey > pQuery->skey))) { - dTrace("QInfo:%p no result in time range %lld-%lld, order %d", pQInfo, pQuery->skey, pQuery->ekey, pQuery->order.order); @@ -3232,14 +3632,31 @@ int32_t vnodeQuerySingleMeterPrepare(SQInfo *pQInfo, SMeterObj *pMeterObj, SMete return TSDB_CODE_SUCCESS; } - /* create runtime environment */ - int32_t ret = setupQueryRuntimeEnv(pMeterObj, pQuery, &pSupporter->runtimeEnv, NULL, pQuery->order.order); + pSupporter->runtimeEnv.pTSBuf = param; + pSupporter->runtimeEnv.cur.vnodeIndex = -1; + if (param != NULL) { + int16_t order = (pQuery->order.order == pSupporter->runtimeEnv.pTSBuf->tsOrder) ? TSQL_SO_ASC : TSQL_SO_DESC; + tsBufSetTraverseOrder(pSupporter->runtimeEnv.pTSBuf, order); + } + + // create runtime environment + int32_t ret = setupQueryRuntimeEnv(pMeterObj, pQuery, &pSupporter->runtimeEnv, NULL, pQuery->order.order, false); if (ret != TSDB_CODE_SUCCESS) { return ret; } vnodeOpenAllFiles(pQInfo, pMeterObj->vnode); + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + if ((ret = allocateOutputBufForGroup(pSupporter, pQuery, false)) != TSDB_CODE_SUCCESS) { + return ret; + } + + pSupporter->runtimeEnv.hashList = taosInitIntHash(10039, sizeof(void *), taosHashInt); + pSupporter->runtimeEnv.usedIndex = 0; + pSupporter->runtimeEnv.pResult = pSupporter->pResult; + } + // in case of last_row query, we set the query timestamp to pMeterObj->lastKey; if (isFirstLastRowQuery(pQuery)) { pQuery->skey = pMeterObj->lastKey; @@ -3269,8 +3686,7 @@ int32_t vnodeQuerySingleMeterPrepare(SQInfo *pQInfo, SMeterObj *pMeterObj, SMete /* * here we set the value for before and after the specified time into the - * parameter for - * interpolation query + * parameter for interpolation query */ pointInterpSupporterSetData(pQInfo, &interpInfo); pointInterpSupporterDestroy(&interpInfo); @@ -3298,6 +3714,7 @@ void vnodeQueryFreeQInfoEx(SQInfo *pQInfo) { return; } + SQuery * pQuery = &pQInfo->query; SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; teardownQueryRuntimeEnv(&pSupporter->runtimeEnv); @@ -3308,9 +3725,16 @@ void vnodeQueryFreeQInfoEx(SQInfo *pQInfo) { pSupporter->pMeterObj = NULL; } - if (pSupporter->pSidSet != NULL) { - for (int32_t i = 0; i < pSupporter->pSidSet->numOfSubSet; ++i) { - destroyBuf(pSupporter->pResult[i].result, pQInfo->query.numOfOutputCols); + if (pSupporter->pSidSet != NULL || isGroupbyNormalCol(pQInfo->query.pGroupbyExpr)) { + int32_t size = 0; + if (isGroupbyNormalCol(pQInfo->query.pGroupbyExpr)) { + size = 10000; + } else if (pSupporter->pSidSet != NULL) { + size = pSupporter->pSidSet->numOfSubSet; + } + + for (int32_t i = 0; i < size; ++i) { + destroyGroupResultBuf(&pSupporter->pResult[i], pQInfo->query.numOfOutputCols); } } @@ -3327,7 +3751,7 @@ void vnodeQueryFreeQInfoEx(SQInfo *pQInfo) { if (pSupporter->pMeterDataInfo != NULL) { for (int32_t j = 0; j < pSupporter->numOfMeters; ++j) { - destroyMeterQueryInfo(pSupporter->pMeterDataInfo[j].pMeterQInfo); + destroyMeterQueryInfo(pSupporter->pMeterDataInfo[j].pMeterQInfo, pQuery->numOfOutputCols); free(pSupporter->pMeterDataInfo[j].pBlock); } } @@ -3338,12 +3762,11 @@ void vnodeQueryFreeQInfoEx(SQInfo *pQInfo) { tfree(pQInfo->pMeterQuerySupporter); } -int32_t vnodeMultiMeterQueryPrepare(SQInfo *pQInfo, SQuery *pQuery) { +int32_t vnodeMultiMeterQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->skey > pQuery->ekey)) || (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->ekey > pQuery->skey))) { - dTrace("QInfo:%p no result in time range %lld-%lld, order %d", pQInfo, pQuery->skey, pQuery->ekey, pQuery->order.order); @@ -3373,7 +3796,7 @@ int32_t vnodeMultiMeterQueryPrepare(SQInfo *pQInfo, SQuery *pQuery) { pSupporter->rawSKey = pQuery->skey; pQuery->lastKey = pQuery->skey; - /* create runtime environment */ + // create runtime environment SSchema *pTagSchema = NULL; tTagSchema *pTagSchemaInfo = pSupporter->pSidSet->pTagSchema; @@ -3381,31 +3804,38 @@ int32_t vnodeMultiMeterQueryPrepare(SQInfo *pQInfo, SQuery *pQuery) { pTagSchema = pTagSchemaInfo->pSchema; } - /* get one queried meter */ + // get one queried meter SMeterObj *pMeter = getMeterObj(pSupporter->pMeterObj, pSupporter->pSidSet->pSids[0]->sid); - int32_t ret = setupQueryRuntimeEnv(pMeter, pQuery, &pSupporter->runtimeEnv, pTagSchema, TSQL_SO_ASC); + pSupporter->runtimeEnv.pTSBuf = param; + pSupporter->runtimeEnv.cur.vnodeIndex = -1; + + // set the ts-comp file traverse order + if (param != NULL) { + int16_t order = (pQuery->order.order == pSupporter->runtimeEnv.pTSBuf->tsOrder) ? TSQL_SO_ASC : TSQL_SO_DESC; + tsBufSetTraverseOrder(pSupporter->runtimeEnv.pTSBuf, order); + } + + int32_t ret = setupQueryRuntimeEnv(pMeter, pQuery, &pSupporter->runtimeEnv, pTagSchema, TSQL_SO_ASC, true); if (ret != TSDB_CODE_SUCCESS) { return ret; } tSidSetSort(pSupporter->pSidSet); - vnodeOpenAllFiles(pQInfo, pMeter->vnode); - pSupporter->pResult = calloc(1, sizeof(SOutputRes) * pSupporter->pSidSet->numOfSubSet); - if (pSupporter->pResult == NULL) { - return TSDB_CODE_SERV_OUT_OF_MEMORY; + + if ((ret = allocateOutputBufForGroup(pSupporter, pQuery, true)) != TSDB_CODE_SUCCESS) { + return ret; } - /* create group result buffer */ - for (int32_t k = 0; k < pSupporter->pSidSet->numOfSubSet; ++k) { - SOutputRes *pOneRes = &pSupporter->pResult[k]; - pOneRes->nAlloc = 1; - pOneRes->result = createInMemGroupResultBuf(pSupporter->runtimeEnv.pCtx, pQuery->numOfOutputCols, pOneRes->nAlloc); + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // group by columns not tags; + pSupporter->runtimeEnv.hashList = taosInitIntHash(10039, sizeof(void *), taosHashInt); + pSupporter->runtimeEnv.usedIndex = 0; + pSupporter->runtimeEnv.pResult = pSupporter->pResult; } if (pQuery->nAggTimeInterval != 0) { - getExtTmpfilePath("/tb_metric_mmap_%lld_%lld_%d_%d", pthread_self(), 0, 0, pSupporter->extBufFile); + getTmpfilePath("tb_metric_mmap", pSupporter->extBufFile); pSupporter->meterOutputFd = open(pSupporter->extBufFile, O_CREAT | O_RDWR, 0666); if (!VALIDFD(pSupporter->meterOutputFd)) { @@ -3429,8 +3859,7 @@ int32_t vnodeMultiMeterQueryPrepare(SQInfo *pQInfo, SQuery *pQuery) { } } - /* metric query do not invoke interpolation, it will be done at the - * second-stage merge */ + // metric query do not invoke interpolation, it will be done at the second-stage merge if (!isPointInterpoQuery(pQuery)) { pQuery->interpoType = TSDB_INTERPO_NONE; } @@ -3471,8 +3900,8 @@ void vnodeDecMeterRefcnt(SQInfo *pQInfo) { * we do not output corresponding information */ num = pSupporter->numOfMeters - num; - dTrace("QInfo:%p metric query is over, dec query ref for %d meters, numOfQueries on %d meters are 0", - pQInfo, pSupporter->numOfMeters, num); + dTrace("QInfo:%p metric query is over, dec query ref for %d meters, numOfQueries on %d meters are 0", pQInfo, + pSupporter->numOfMeters, num); } } @@ -3523,7 +3952,7 @@ TSKEY getTimestampInDiskBlock(SQueryRuntimeEnv *pRuntimeEnv, int32_t index) { if (!vnodeIsDatablockLoaded(pRuntimeEnv, pMeterObj, fileIndex)) { dTrace("QInfo:%p vid:%d sid:%d id:%s, fileId:%d, slot:%d load data block due to primary key required", - GET_QINFO_ADDR(pQuery), pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->fileId, pQuery->slot); + GET_QINFO_ADDR(pQuery), pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->fileId, pQuery->slot); // todo handle failed to load data, file corrupted // todo refactor the return value @@ -3532,6 +3961,11 @@ TSKEY getTimestampInDiskBlock(SQueryRuntimeEnv *pRuntimeEnv, int32_t index) { UNUSED(ret); } + // the fields info is not loaded, load it into memory + if (pQuery->pFields == NULL || pQuery->pFields[pQuery->slot] == NULL) { + loadDataBlockFieldsInfo(pRuntimeEnv, &pRuntimeEnv->pHeaderFiles[fileIndex], pBlock, &pQuery->pFields[pQuery->slot]); + } + SET_DATA_BLOCK_LOADED(pRuntimeEnv->blockStatus); SET_FILE_BLOCK_FLAG(pRuntimeEnv->blockStatus); @@ -3561,13 +3995,15 @@ static void getFirstDataBlockInCache(SQueryRuntimeEnv *pRuntimeEnv) { } } +//TODO handle case that the cache is allocated but not assign to SMeterObj void getQueryPositionForCacheInvalid(SQueryRuntimeEnv *pRuntimeEnv, __block_search_fn_t searchFn) { SQuery * pQuery = pRuntimeEnv->pQuery; SQInfo * pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery); SMeterObj *pMeterObj = pRuntimeEnv->pMeterObj; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - dTrace("QInfo:%p vid:%d sid:%d id:%s cache block re-allocated to other meter, " + dTrace( + "QInfo:%p vid:%d sid:%d id:%s cache block re-allocated to other meter, " "try get query start position in file/cache, qrange:%lld-%lld, lastKey:%lld", pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey, pQuery->lastKey); @@ -3578,8 +4014,8 @@ void getQueryPositionForCacheInvalid(SQueryRuntimeEnv *pRuntimeEnv, __block_sear */ bool ret = getQualifiedDataBlock(pMeterObj, pRuntimeEnv, QUERY_RANGE_LESS_EQUAL, searchFn); - dTrace("QInfo:%p vid:%d sid:%d id:%s find the possible position, fileId:%d, slot:%d, pos:%d", - pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->fileId, pQuery->slot, pQuery->pos); + dTrace("QInfo:%p vid:%d sid:%d id:%s find the possible position, fileId:%d, slot:%d, pos:%d", pQInfo, + pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->fileId, pQuery->slot, pQuery->pos); if (ret) { TSKEY key = getTimestampInDiskBlock(pRuntimeEnv, pQuery->pos); @@ -3594,8 +4030,8 @@ void getQueryPositionForCacheInvalid(SQueryRuntimeEnv *pRuntimeEnv, __block_sear } else { bool ret = getQualifiedDataBlock(pMeterObj, pRuntimeEnv, QUERY_RANGE_GREATER_EQUAL, searchFn); if (ret) { - dTrace("QInfo:%p vid:%d sid:%d id:%s find the possible position, fileId:%d, slot:%d, pos:%d", - pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->fileId, pQuery->slot, pQuery->pos); + dTrace("QInfo:%p vid:%d sid:%d id:%s find the possible position, fileId:%d, slot:%d, pos:%d", pQInfo, + pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->fileId, pQuery->slot, pQuery->pos); TSKEY key = getTimestampInDiskBlock(pRuntimeEnv, pQuery->pos); @@ -3610,8 +4046,8 @@ void getQueryPositionForCacheInvalid(SQueryRuntimeEnv *pRuntimeEnv, __block_sear */ getFirstDataBlockInCache(pRuntimeEnv); - dTrace("QInfo:%p vid:%d sid:%d id:%s find the new position in cache, fileId:%d, slot:%d, pos:%d", - pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->fileId, pQuery->slot, pQuery->pos); + dTrace("QInfo:%p vid:%d sid:%d id:%s find the new position in cache, fileId:%d, slot:%d, pos:%d", pQInfo, + pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->fileId, pQuery->slot, pQuery->pos); } } } @@ -3650,8 +4086,9 @@ static int32_t moveToNextBlockInCache(SQueryRuntimeEnv *pRuntimeEnv, int32_t ste setQueryStatus(pQuery, QUERY_COMPLETED); } - //the skip operation does NOT set the start position - //assert(pRuntimeEnv->startPos.fileId < 0); + // the skip operation does NOT set the startPos yet + // assert(pRuntimeEnv->startPos.fileId < 0); + } else { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); } @@ -3738,7 +4175,6 @@ static int32_t moveToNextBlock(SQueryRuntimeEnv *pRuntimeEnv, int32_t step, __bl int32_t ret = LoadDatablockOnDemand(&pQuery->pBlock[pQuery->slot], &pQuery->pFields[pQuery->slot], &pRuntimeEnv->blockStatus, pRuntimeEnv, fileIndex, pQuery->slot, searchFn, true); - if (ret != DISK_DATA_LOADED) { /* * if it is the last block of file, set current access position at the last point of the meter in this file, @@ -3755,8 +4191,8 @@ static int32_t moveToNextBlock(SQueryRuntimeEnv *pRuntimeEnv, int32_t step, __bl static void doHandleFileBlockImpl(SQueryRuntimeEnv *pRuntimeEnv, SBlockInfo *pblockInfo, __block_search_fn_t searchFn, SData **sdata, int32_t *numOfRes, int32_t blockLoadStatus, int32_t *forwardStep) { - SQuery * pQuery = pRuntimeEnv->pQuery; - SQueryCostStatistics *pSummary = &pRuntimeEnv->summary; + SQuery * pQuery = pRuntimeEnv->pQuery; + SQueryCostSummary *pSummary = &pRuntimeEnv->summary; int64_t start = taosGetTimestampUs(); @@ -3777,9 +4213,9 @@ static void doHandleFileBlockImpl(SQueryRuntimeEnv *pRuntimeEnv, SBlockInfo *pbl static void doHandleCacheBlockImpl(SQueryRuntimeEnv *pRuntimeEnv, SBlockInfo *pblockInfo, __block_search_fn_t searchFn, int32_t *numOfRes, int32_t *forwardStep) { - SQuery * pQuery = pRuntimeEnv->pQuery; - SMeterObj * pMeterObj = pRuntimeEnv->pMeterObj; - SQueryCostStatistics *pSummary = &pRuntimeEnv->summary; + SQuery * pQuery = pRuntimeEnv->pQuery; + SMeterObj * pMeterObj = pRuntimeEnv->pMeterObj; + SQueryCostSummary *pSummary = &pRuntimeEnv->summary; int64_t start = taosGetTimestampUs(); @@ -3823,9 +4259,9 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SMeterObj *pMeterObj = pRuntimeEnv->pMeterObj; SData ** sdata = pRuntimeEnv->colDataBuffer; - __block_search_fn_t searchFn = vnodeSearchKeyFunc[pMeterObj->searchAlgorithm]; - int32_t blockLoadStatus = DISK_DATA_LOADED; - SQueryCostStatistics *pSummary = &pRuntimeEnv->summary; + __block_search_fn_t searchFn = vnodeSearchKeyFunc[pMeterObj->searchAlgorithm]; + int32_t blockLoadStatus = DISK_DATA_LOADED; + SQueryCostSummary * pSummary = &pRuntimeEnv->summary; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); @@ -3834,8 +4270,8 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { assert(pQuery->slot == pStartPos->slot); dTrace("QInfo:%p query start, qrange:%lld-%lld, lastkey:%lld, order:%d, start fileId:%d, slot:%d, pos:%d, bstatus:%d", - GET_QINFO_ADDR(pQuery), pQuery->skey, pQuery->ekey, pQuery->lastKey, pQuery->order.order, pStartPos->fileId, - pStartPos->slot, pStartPos->pos, pRuntimeEnv->blockStatus); + GET_QINFO_ADDR(pQuery), pQuery->skey, pQuery->ekey, pQuery->lastKey, pQuery->order.order, pStartPos->fileId, + pStartPos->slot, pStartPos->pos, pRuntimeEnv->blockStatus); while (1) { // check if query is killed or not set the status of query to pass the status check @@ -3854,8 +4290,8 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { } dTrace("QInfo:%p check data block, brange:%lld-%lld, fileId:%d, slot:%d, pos:%d, bstatus:%d, rows:%d, checked:%d", - GET_QINFO_ADDR(pQuery), blockInfo.keyFirst, blockInfo.keyLast, pQuery->fileId, pQuery->slot, pQuery->pos, - pRuntimeEnv->blockStatus, blockInfo.size, forwardStep); + GET_QINFO_ADDR(pQuery), blockInfo.keyFirst, blockInfo.keyLast, pQuery->fileId, pQuery->slot, pQuery->pos, + pRuntimeEnv->blockStatus, blockInfo.size, forwardStep); // save last access position int32_t accessPos = pQuery->pos + (forwardStep - 1) * step; @@ -3873,15 +4309,6 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { */ if (pQuery->nAggTimeInterval > 0 || (Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL) && pQuery->checkBufferInLoop == 1)) { - // if (IS_DISK_DATA_BLOCK(pQuery)) { - // getNextQueryStartPos(&pRuntimeEnv->nextPos, - // pQuery->pBlock, pQuery->slot, accessPos, - // pRuntimeEnv); - // } else { - // getNextQueryStartPosInCache(&pRuntimeEnv->nextPos, - // pData, pQuery->slot, accessPos, pRuntimeEnv); - // } - if (nextPos >= blockInfo.size || nextPos < 0) { moveToNextBlock(pRuntimeEnv, step, searchFn, !LOAD_DATA); @@ -3945,8 +4372,12 @@ void queryOnBlock(SMeterQuerySupportObj *pSupporter, int64_t *primaryKeys, int32 pSupporter->pResult[pDataHeadInfoEx->groupIdx].numOfRows = numOfRes; } - /* used to decide the correct start position in cache after check all data in files */ + // used to decide the correct start position in cache after check all data in files updatelastkey(pQuery, pDataHeadInfoEx->pMeterQInfo); + if (pRuntimeEnv->pTSBuf != NULL) { + pDataHeadInfoEx->pMeterQInfo->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + } + } else { applyIntervalQueryOnBlock(pSupporter, pDataHeadInfoEx, data, primaryKeys, pBlockBasicInfo, blockStatus, pFields, searchFn); @@ -3967,39 +4398,78 @@ static void doSetTagValueInParam(tTagSchema *pTagSchema, int32_t tagColIdx, SMet SSchema *pCol = &pTagSchema->pSchema[tagColIdx]; tVariantDestroy(param); - tVariantCreateB(param, pStr, pCol->bytes, pCol->type); + tVariantCreateFromBinary(param, pStr, pCol->bytes, pCol->type); + + if (isNull(pStr, pCol->type)) { + param->nType = TSDB_DATA_TYPE_NULL; + } } -void vnodeSetTagValueInParam(tSidSet *pSidSet, SQueryRuntimeEnv *pRuntimeEnv, SMeterSidExtInfo *pMeterInfo) { +void vnodeSetTagValueInParam(tSidSet *pSidSet, SQueryRuntimeEnv *pRuntimeEnv, SMeterSidExtInfo *pMeterSidInfo) { SQuery * pQuery = pRuntimeEnv->pQuery; tTagSchema *pTagSchema = pSidSet->pTagSchema; - // set tag value, by which the results are aggregated. - for (int32_t idx = 0; idx < pQuery->numOfOutputCols; ++idx) { - SColIndexEx *pColEx = &pQuery->pSelectExpr[idx].pBase.colInfo; - if (!pColEx->isTag) { - continue; + SSqlFuncExprMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; + if (pQuery->numOfOutputCols == 1 && pFuncMsg->functionId == TSDB_FUNC_TS_COMP) { + assert(pFuncMsg->numOfParams == 1); + doSetTagValueInParam(pTagSchema, pFuncMsg->arg->argValue.i64, pMeterSidInfo, &pRuntimeEnv->pCtx[0].tag); + } else { + // set tag value, by which the results are aggregated. + for (int32_t idx = 0; idx < pQuery->numOfOutputCols; ++idx) { + SColIndexEx *pColEx = &pQuery->pSelectExpr[idx].pBase.colInfo; + + // ts_comp column required the tag value for join filter + if (!TSDB_COL_IS_TAG(pColEx->flag)) { + continue; + } + + doSetTagValueInParam(pTagSchema, pColEx->colIdx, pMeterSidInfo, &pRuntimeEnv->pCtx[idx].tag); } - doSetTagValueInParam(pTagSchema, pColEx->colIdx, pMeterInfo, &pRuntimeEnv->pCtx[idx].intermediateBuf[3]); + // set the join tag for first column + SSqlFuncExprMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; + if (pFuncMsg->functionId == TSDB_FUNC_TS && pFuncMsg->colInfo.colIdx == PRIMARYKEY_TIMESTAMP_COL_INDEX && + pRuntimeEnv->pTSBuf != NULL) { + assert(pFuncMsg->numOfParams == 1); + doSetTagValueInParam(pTagSchema, pFuncMsg->arg->argValue.i64, pMeterSidInfo, &pRuntimeEnv->pCtx[0].tag); + } } } -static void doMerge(SQuery *pQuery, SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, tFilePage *inputSrc, - int32_t inputIdx, int16_t *offset, int32_t maxRow, bool mergeFlag) { +static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, tFilePage *inputSrc, int32_t inputIdx, + bool mergeFlag) { + SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t funcId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (!mergeFlag) { pCtx[i].aOutputBuf = pCtx[i].aOutputBuf + pCtx[i].outputBytes; - aAggs[funcId].init(&pCtx[i]); + pCtx[i].currentStage = FIRST_STAGE_MERGE; + + resetResultInfo(pCtx[i].resultInfo); + aAggs[functionId].init(&pCtx[i]); } - pCtx[i].hasNullValue = true; + + pCtx[i].hasNull = true; pCtx[i].nStartQueryTimestamp = timestamp; - pCtx[i].aInputElemBuf = inputSrc->data + (offset[i] * maxRow) + pCtx[i].outputBytes * inputIdx; + pCtx[i].aInputElemBuf = ((char *) inputSrc->data) + + ((int32_t) pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage) + pCtx[i].outputBytes * inputIdx; - aAggs[funcId].distMergeFunc(&pCtx[i]); + //in case of tag column, the tag information should be extracted from input buffer + if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) { + tVariantDestroy(&pCtx[i].tag); + tVariantCreateFromBinary(&pCtx[i].tag, pCtx[i].aInputElemBuf, pCtx[i].inputBytes, pCtx[i].inputType); + } + } + + for(int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (functionId == TSDB_FUNC_TAG_DUMMY) { + continue; + } + + aAggs[functionId].distMergeFunc(&pCtx[i]); } } @@ -4031,15 +4501,15 @@ static void printBinaryData(int32_t functionId, char *data, int32_t srcDataType) printf("%ld,%lf\t", *(TSKEY *)data, *(double *)(data + TSDB_KEYSIZE + 1)); break; } - } else if (functionId == TSDB_FUNC_AVG_DST) { + } else if (functionId == TSDB_FUNC_AVG) { printf("%lf,%d\t", *(double *)data, *(int32_t *)(data + sizeof(double))); - } else if (functionId == TSDB_FUNC_SPREAD_DST) { + } else if (functionId == TSDB_FUNC_SPREAD) { printf("%lf,%lf\t", *(double *)data, *(double *)(data + sizeof(double))); - } else if (functionId == TSDB_FUNC_WAVG_DST) { + } else if (functionId == TSDB_FUNC_TWA) { data += 1; printf("%lf,%ld,%ld,%ld\t", *(double *)data, *(int64_t *)(data + 8), *(int64_t *)(data + 16), *(int64_t *)(data + 24)); - } else if (functionId == TSDB_FUNC_MIN_DST || functionId == TSDB_FUNC_MAX_DST) { + } else if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) { switch (srcDataType) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_BOOL: @@ -4062,7 +4532,7 @@ static void printBinaryData(int32_t functionId, char *data, int32_t srcDataType) printf("%f\t", *(float *)data); break; } - } else if (functionId == TSDB_FUNC_SUM_DST) { + } else if (functionId == TSDB_FUNC_SUM) { if (srcDataType == TSDB_DATA_TYPE_FLOAT || srcDataType == TSDB_DATA_TYPE_DOUBLE) { printf("%lf\t", *(float *)data); } else { @@ -4087,7 +4557,7 @@ void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOf int32_t colIdx = pQuery->pSelectExpr[i].pBase.colInfo.colIdx; int32_t type = 0; - if (pQuery->pSelectExpr[i].pBase.colInfo.isTag) { + if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[i].pBase.colInfo.flag)) { type = pQuery->pSelectExpr[i].resType; } else { type = pMeterObj->schema[colIdx].type; @@ -4202,8 +4672,8 @@ int32_t mergeMetersResultToOneGroups(SMeterQuerySupportObj *pSupporter) { dTrace("QInfo:%p no result in group %d, continue", GET_QINFO_ADDR(pQuery), pSupporter->subgroupIdx - 1); } - dTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%lldms", - GET_QINFO_ADDR(pQuery), pSupporter->subgroupIdx - 1, pSupporter->pSidSet->numOfSubSet, taosGetTimestampMs() - st); + dTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%lldms", GET_QINFO_ADDR(pQuery), + pSupporter->subgroupIdx - 1, pSupporter->pSidSet->numOfSubSet, taosGetTimestampMs() - st); return pSupporter->numOfGroupResultPages; } @@ -4240,8 +4710,8 @@ void copyResToQueryResultBuf(SMeterQuerySupportObj *pSupporter, SQuery *pQuery) int32_t doMergeMetersResultsToGroupRes(SMeterQuerySupportObj *pSupporter, SQuery *pQuery, SQueryRuntimeEnv *pRuntimeEnv, SMeterDataInfo *pMeterHeadDataInfo, int32_t start, int32_t end) { + // calculate the maximum required space if (pSupporter->groupResultSize == 0) { - /* calculate the maximum required space */ for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { pSupporter->groupResultSize += sizeof(tFilePage) + pQuery->pointsToRead * pRuntimeEnv->pCtx[i].outputBytes; } @@ -4268,7 +4738,7 @@ int32_t doMergeMetersResultsToGroupRes(SMeterQuerySupportObj *pSupporter, SQuery return 0; } - SCompSupporter cs = {pValidMeter, posArray, pSupporter}; // 1 == ascending order + SCompSupporter cs = {pValidMeter, posArray, pSupporter}; SLoserTreeInfo *pTree = NULL; tLoserTreeCreate(&pTree, numOfMeters, &cs, meterResultComparator); @@ -4287,8 +4757,7 @@ int32_t doMergeMetersResultsToGroupRes(SMeterQuerySupportObj *pSupporter, SQuery int64_t ts = getCurrentTimestamp(&cs, pos); if (ts == lastTimestamp) { // merge with the last one - doMerge(pQuery, pRuntimeEnv, ts, pPage, position->rowIdx, pRuntimeEnv->offset, pRuntimeEnv->numOfRowsPerPage, - true); + doMerge(pRuntimeEnv, ts, pPage, position->rowIdx, true); } else { // copy data to disk buffer if (buffer[0]->numOfElems == pQuery->pointsToRead) { @@ -4300,8 +4769,7 @@ int32_t doMergeMetersResultsToGroupRes(SMeterQuerySupportObj *pSupporter, SQuery if (pPage->numOfElems <= 0) { // current source data page is empty // do nothing } else { - doMerge(pQuery, pRuntimeEnv, ts, pPage, position->rowIdx, pRuntimeEnv->offset, pRuntimeEnv->numOfRowsPerPage, - false); + doMerge(pRuntimeEnv, ts, pPage, position->rowIdx, false); buffer[0]->numOfElems += 1; } } @@ -4312,25 +4780,22 @@ int32_t doMergeMetersResultsToGroupRes(SMeterQuerySupportObj *pSupporter, SQuery cs.pPosition[pos].rowIdx = 0; cs.pPosition[pos].pageIdx += 1; // try next page - /*check if current page is empty or not. if it is empty, ignore it and try - * next*/ + // check if current page is empty or not. if it is empty, ignore it and try next if (cs.pPosition[pos].pageIdx <= cs.pInfoEx[pos]->pMeterQInfo->numOfPages - 1) { tFilePage *newPage = getMeterDataPage(cs.pSupporter, pValidMeter[pos], position->pageIdx); if (newPage->numOfElems <= 0) { - /* if current source data page is null, it must be the last page of - * source output page */ + // if current source data page is null, it must be the last page of source output page cs.pPosition[pos].pageIdx += 1; assert(cs.pPosition[pos].pageIdx >= cs.pInfoEx[pos]->pMeterQInfo->numOfPages - 1); } } - /* the following code must be executed if current source pages are - * exhausted */ + // the following code must be executed if current source pages are exhausted if (cs.pPosition[pos].pageIdx >= cs.pInfoEx[pos]->pMeterQInfo->numOfPages) { cs.pPosition[pos].pageIdx = -1; cs.pPosition[pos].rowIdx = -1; - /* all input sources are exhausted */ + // all input sources are exhausted if (--numOfMeters == 0) { break; } @@ -4365,11 +4830,11 @@ int32_t doMergeMetersResultsToGroupRes(SMeterQuerySupportObj *pSupporter, SQuery static void extendDiskBuf(SMeterQuerySupportObj *pSupporter, int32_t numOfPages) { assert(pSupporter->numOfPages * DEFAULT_INTERN_BUF_SIZE == pSupporter->bufSize); - munmap(pSupporter->meterOutputMMapBuf, pSupporter->bufSize); + int32_t ret = munmap(pSupporter->meterOutputMMapBuf, pSupporter->bufSize); pSupporter->numOfPages = numOfPages; // disk-based output buffer is exhausted, try to extend the disk-based buffer - int32_t ret = ftruncate(pSupporter->meterOutputFd, pSupporter->numOfPages * DEFAULT_INTERN_BUF_SIZE); + ret = ftruncate(pSupporter->meterOutputFd, pSupporter->numOfPages * DEFAULT_INTERN_BUF_SIZE); if (ret != 0) { perror("error in allocate the disk-based buffer"); return; @@ -4412,7 +4877,6 @@ void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx) { pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes; pCtx[k].size = 1; pCtx[k].startOffset = 0; - pCtx[k].numOfIteratedElems = 0; pQuery->sdata[k]->len = 0; } } @@ -4444,18 +4908,42 @@ void doCloseAllOpenedResults(SMeterQuerySupportObj *pSupporter) { } } -void disableFunctForSuppleScanAndSetSortOrder(SQueryRuntimeEnv *pRuntimeEnv, int32_t order) { +void disableFunctForSuppleScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - pRuntimeEnv->pCtx[i].order = (pRuntimeEnv->pCtx[i].order) ^ 1; - int32_t functId = pQuery->pSelectExpr[i].pBase.functionId; + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + pRuntimeEnv->pCtx[i].order = (pRuntimeEnv->pCtx[i].order) ^ 1; + } + + for (int32_t i = 0; i < pRuntimeEnv->usedIndex; ++i) { + SOutputRes *buf = &pRuntimeEnv->pResult[i]; - if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSQL_SO_DESC) || - ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSQL_SO_ASC)) { - pRuntimeEnv->go[i] = true; - } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { - pRuntimeEnv->go[i] = false; + // open/close the specified query for each group result + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; + + if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSQL_SO_DESC) || + ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSQL_SO_ASC)) { + buf->resultInfo[j].complete = false; + } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { + buf->resultInfo[j].complete = true; + } + } + } + } else { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + pRuntimeEnv->pCtx[i].order = (pRuntimeEnv->pCtx[i].order) ^ 1; + int32_t functId = pQuery->pSelectExpr[i].pBase.functionId; + + SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]); + if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSQL_SO_DESC) || + ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSQL_SO_ASC)) { + pResInfo->complete = false; + + } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { + pResInfo->complete = true; + } } } @@ -4467,27 +4955,54 @@ void enableFunctForMasterScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { pRuntimeEnv->pCtx[i].order = (pRuntimeEnv->pCtx[i].order) ^ 1; - pRuntimeEnv->go[i] = true; } pQuery->order.order = (pQuery->order.order ^ 1); } -tFilePage **createInMemGroupResultBuf(SQLFunctionCtx *pCtx, int32_t nOutputCols, int32_t nAlloc) { - tFilePage **pTempBuf = malloc(POINTER_BYTES * nOutputCols); +void createGroupResultBuf(SQuery *pQuery, SOutputRes *pOneResult, bool isMetricQuery) { + int32_t numOfOutput = pQuery->numOfOutputCols; + + pOneResult->resultInfo = calloc((size_t)numOfOutput, sizeof(SResultInfo)); + + pOneResult->result = malloc(POINTER_BYTES * numOfOutput); + for (int32_t i = 0; i < numOfOutput; ++i) { + size_t size = pQuery->pSelectExpr[i].interResBytes; + SResultInfo *pResInfo = &pOneResult->resultInfo[i]; + + pOneResult->result[i] = malloc(sizeof(tFilePage) + size * pOneResult->nAlloc); + pOneResult->result[i]->numOfElems = 0; + + setResultInfoBuf(pResInfo, (int32_t)size, isMetricQuery); + } +} + +void clearGroupResultBuf(SOutputRes *pOneOutputRes, int32_t nOutputCols) { + if (pOneOutputRes == NULL) { + return; + } + for (int32_t i = 0; i < nOutputCols; ++i) { - pTempBuf[i] = malloc(sizeof(tFilePage) + pCtx[i].outputBytes * nAlloc); - pTempBuf[i]->numOfElems = 0; + SResultInfo *pResInfo = &pOneOutputRes->resultInfo[i]; + int32_t size = sizeof(tFilePage) + pResInfo->bufLen * pOneOutputRes->nAlloc; + + memset(pOneOutputRes->result[i], 0, (size_t)size); + resetResultInfo(pResInfo); } - return pTempBuf; } -void destroyBuf(tFilePage **pBuf, int32_t nOutputCols) { +void destroyGroupResultBuf(SOutputRes *pOneOutputRes, int32_t nOutputCols) { + if (pOneOutputRes == NULL) { + return; + } + for (int32_t i = 0; i < nOutputCols; ++i) { - free(pBuf[i]); + free(pOneOutputRes->result[i]); + free(pOneOutputRes->resultInfo[i].interResultBuf); } - free(pBuf); + free(pOneOutputRes->resultInfo); + free(pOneOutputRes->result); } void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { @@ -4495,20 +5010,32 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { int32_t rows = pRuntimeEnv->pMeterObj->pointsPerFileBlock; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - pRuntimeEnv->pCtx[i].numOfOutputElems = 0; - if (QUERY_IS_ASC_QUERY(pQuery)) { - pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data; + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + + // ts_comp query does not required reversed output + if (QUERY_IS_ASC_QUERY(pQuery) || isTSCompQuery(pQuery)) { + pCtx->aOutputBuf = pQuery->sdata[i]->data; } else { // point to the last position of output buffer for desc query - pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + (rows - 1) * pRuntimeEnv->pCtx[i].outputBytes; + pCtx->aOutputBuf = pQuery->sdata[i]->data + (rows - 1) * pCtx->outputBytes; } - } - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + /* + * set the output buffer information and intermediate buffer + * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. + */ + resetResultInfo(&pRuntimeEnv->resultInfo[i]); + pCtx->resultInfo = &pRuntimeEnv->resultInfo[i]; + + // set the timestamp output buffer for top/bottom/diff query int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; + pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } + + memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].resBytes * rows); } + + initCtxOutputBuf(pRuntimeEnv); } void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { @@ -4535,6 +5062,8 @@ void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { */ pRuntimeEnv->pCtx[j].ptsOutputBuf += TSDB_KEYSIZE * output * factor; } + + resetResultInfo(pRuntimeEnv->pCtx[j].resultInfo); } } @@ -4543,19 +5072,11 @@ void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + pRuntimeEnv->pCtx[j].currentStage = 0; aAggs[functionId].init(&pRuntimeEnv->pCtx[j]); } } -void cleanCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - int32_t rows = pRuntimeEnv->pMeterObj->pointsPerFileBlock; - - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].resBytes * rows); - } -} - void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; if (pQuery->pointsRead == 0 || pQuery->limit.offset == 0) { @@ -4613,7 +5134,11 @@ void moveDescOrderResultsToFront(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; int32_t maxrows = pQuery->pointsToRead; - if (pQuery->pointsRead > 0 && pQuery->pointsRead < maxrows && !QUERY_IS_ASC_QUERY(pQuery)) { + if (QUERY_IS_ASC_QUERY(pQuery) || isTSCompQuery(pQuery)) { + return; + } + + if (pQuery->pointsRead > 0 && pQuery->pointsRead < maxrows) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; memmove(pQuery->sdata[i]->data, pQuery->sdata[i]->data + (maxrows - pQuery->pointsRead) * bytes, @@ -4631,6 +5156,8 @@ typedef struct SQueryStatus { TSKEY ekey; int8_t overStatus; TSKEY lastKey; + + STSCursor cur; } SQueryStatus; static void queryStatusSave(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus *pStatus) { @@ -4643,6 +5170,13 @@ static void queryStatusSave(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus *pStatus pStatus->next = pRuntimeEnv->nextPos; pStatus->end = pRuntimeEnv->endPos; + pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); // save the cursor + + if (pRuntimeEnv->pTSBuf) { + pRuntimeEnv->pTSBuf->cur.order ^= 1; + tsBufNextPos(pRuntimeEnv->pTSBuf); + } + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); SWAP(pQuery->skey, pQuery->ekey, TSKEY); @@ -4661,9 +5195,11 @@ static void queryStatusRestore(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus *pSta pRuntimeEnv->startPos = pStatus->start; pRuntimeEnv->nextPos = pStatus->next; pRuntimeEnv->endPos = pStatus->end; + + tsBufSetCursor(pRuntimeEnv->pTSBuf, &pStatus->cur); } -static void doSupplementaryScan(SQueryRuntimeEnv *pRuntimeEnv) { +static void doSingleMeterSupplementScan(SQueryRuntimeEnv *pRuntimeEnv) { SQuery * pQuery = pRuntimeEnv->pQuery; SQueryStatus qStatus = {0}; @@ -4678,8 +5214,8 @@ static void doSupplementaryScan(SQueryRuntimeEnv *pRuntimeEnv) { assert((QUERY_IS_ASC_QUERY(pQuery) && endKey <= pQuery->ekey) || (!QUERY_IS_ASC_QUERY(pQuery) && endKey >= pQuery->ekey)); - /* close necessary function execution during supplementary scan */ - disableFunctForSuppleScanAndSetSortOrder(pRuntimeEnv, pQuery->order.order); + // close necessary function execution during supplementary scan + disableFunctForSuppleScan(pRuntimeEnv, pQuery->order.order); queryStatusSave(pRuntimeEnv, &qStatus); doScanAllDataBlocks(pRuntimeEnv); @@ -4714,19 +5250,36 @@ void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { while (1) { doScanAllDataBlocks(pRuntimeEnv); - /* applied to agg functions (e.g., stddev) */ - bool more = false; - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { - pRuntimeEnv->go[j] = aAggs[pQuery->pSelectExpr[j].pBase.functionId].xNextStep(&pRuntimeEnv->pCtx[j]); - more |= pRuntimeEnv->go[j]; + // applied to agg functions (e.g., stddev) + bool toContinue = true; + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + // for each group result, call the finalize function for each column + for (int32_t i = 0; i < pRuntimeEnv->usedIndex; ++i) { + SOutputRes *buf = &pRuntimeEnv->pResult[i]; + setGroupOutputBuffer(pRuntimeEnv, buf); + + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + aAggs[pQuery->pSelectExpr[j].pBase.functionId].xNextStep(&pRuntimeEnv->pCtx[j]); + SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + + toContinue &= (pResInfo->complete); + } + } + } else { + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + aAggs[pQuery->pSelectExpr[j].pBase.functionId].xNextStep(&pRuntimeEnv->pCtx[j]); + SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + + toContinue &= (pResInfo->complete); + } } - if (!more) { + if (toContinue) { break; } - // set the correct start position, and load the corresponding block in - // buffer if required. + // set the correct start position, and load the corresponding block in buffer if required. TSKEY actKey = loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->startPos); assert((QUERY_IS_ASC_QUERY(pQuery) && actKey >= pQuery->skey) || (!QUERY_IS_ASC_QUERY(pQuery) && actKey <= pQuery->skey)); @@ -4741,24 +5294,70 @@ void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { } } - doSupplementaryScan(pRuntimeEnv); + doSingleMeterSupplementScan(pRuntimeEnv); - /* reset status code */ - memset(pRuntimeEnv->go, true, (size_t)pQuery->numOfOutputCols); + // reset status code + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + for (int32_t i = 0; i < pRuntimeEnv->usedIndex; ++i) { + SOutputRes *buf = &pRuntimeEnv->pResult[i]; + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + buf->resultInfo[j].complete = false; + } + } + } else { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]); + if (pResInfo != NULL) { + pResInfo->complete = false; + } + } + } } void doFinalizeResult(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { - aAggs[pQuery->pSelectExpr[j].pBase.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + // for each group result, call the finalize function for each column + for (int32_t i = 0; i < pRuntimeEnv->usedIndex; ++i) { + SOutputRes *buf = &pRuntimeEnv->pResult[i]; + setGroupOutputBuffer(pRuntimeEnv, buf); + + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + aAggs[pQuery->pSelectExpr[j].pBase.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); + } + + /* + * set the number of output results for group by normal columns, the number of output rows usually is 1 except + * the top and bottom query + */ + buf->numOfRows = getNumOfResult(pRuntimeEnv); + } + + } else { + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + aAggs[pQuery->pSelectExpr[j].pBase.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); + } } } +static bool hasMainOutput(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + + if (functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TAGPRJ) { + return true; + } + } + + return false; +} + int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; + bool hasMainFunction = hasMainOutput(pQuery); int64_t maxOutput = 0; - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; @@ -4766,14 +5365,17 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { * ts, tag, tagprj function can not decide the output number of current query * the number of output result is decided by main output */ - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { + if (hasMainFunction && + (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ)) { continue; } - if (maxOutput < pRuntimeEnv->pCtx[j].numOfOutputElems) { - maxOutput = pRuntimeEnv->pCtx[j].numOfOutputElems; + SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) { + maxOutput = pResInfo->numOfRes; } } + return maxOutput; } @@ -4858,8 +5460,8 @@ SMeterDataInfo **vnodeFilterQualifiedMeters(SQInfo *pQInfo, int32_t vid, SQueryF char * pHeaderData = pQueryFileInfo->pHeaderFileData; int32_t tmsize = sizeof(SCompHeader) * (pVnode->cfg.maxSessions) + sizeof(TSCKSUM); + // file is corrupted, abort query in current file if (validateHeaderOffsetSegment(pQInfo, pQueryFileInfo->headerFilePath, vid, pHeaderData, tmsize) < 0) { - /* file is corrupted, abort query in current file */ *numOfMeters = 0; return 0; } @@ -4911,7 +5513,6 @@ SMeterDataInfo **vnodeFilterQualifiedMeters(SQInfo *pQInfo, int32_t vid, SQueryF int64_t headerOffset = TSDB_FILE_HEADER_LEN + sizeof(SCompHeader) * pMeterObj->sid; SCompHeader *compHeader = (SCompHeader *)(pHeaderData + headerOffset); - comp_block_info_read_bytes += sizeof(SCompHeader); if (compHeader->compInfoOffset == 0) { continue; @@ -4925,7 +5526,10 @@ SMeterDataInfo **vnodeFilterQualifiedMeters(SQInfo *pQInfo, int32_t vid, SQueryF } pOneMeterDataInfo->offsetInHeaderFile = (uint64_t)compHeader->compInfoOffset; - setMeterQueryInfo(pSupporter, pOneMeterDataInfo); + + if (pOneMeterDataInfo->pMeterQInfo == NULL) { + pOneMeterDataInfo->pMeterQInfo = createMeterQueryInfo(pQuery, pSupporter->rawSKey, pSupporter->rawEKey); + } pReqMeterDataInfo[*numOfMeters] = pOneMeterDataInfo; (*numOfMeters) += 1; @@ -4941,41 +5545,111 @@ SMeterDataInfo **vnodeFilterQualifiedMeters(SQInfo *pQInfo, int32_t vid, SQueryF return pReqMeterDataInfo; } -void setMeterQueryInfo(SMeterQuerySupportObj *pSupporter, SMeterDataInfo *pMeterDataInfo) { - if (pMeterDataInfo->pMeterQInfo != NULL) { +SMeterQueryInfo *createMeterQueryInfo(SQuery *pQuery, TSKEY skey, TSKEY ekey) { + SMeterQueryInfo *pMeterQueryInfo = calloc(1, sizeof(SMeterQueryInfo)); + + pMeterQueryInfo->skey = skey; + pMeterQueryInfo->ekey = ekey; + pMeterQueryInfo->lastKey = skey; + + pMeterQueryInfo->numOfPages = 0; + pMeterQueryInfo->numOfAlloc = INIT_ALLOCATE_DISK_PAGES; + pMeterQueryInfo->pageList = calloc(pMeterQueryInfo->numOfAlloc, sizeof(uint32_t)); + pMeterQueryInfo->lastResRows = 0; + + pMeterQueryInfo->cur.vnodeIndex = -1; + + pMeterQueryInfo->resultInfo = calloc((size_t)pQuery->numOfOutputCols, sizeof(SResultInfo)); + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SResultInfo *pResInfo = &pMeterQueryInfo->resultInfo[i]; + setResultInfoBuf(pResInfo, pQuery->pSelectExpr[i].interResBytes, true); + } + + return pMeterQueryInfo; +} + +void destroyMeterQueryInfo(SMeterQueryInfo *pMeterQueryInfo, int32_t numOfCols) { + if (pMeterQueryInfo == NULL) { + return; + } + + free(pMeterQueryInfo->pageList); + for (int32_t i = 0; i < numOfCols; ++i) { + tfree(pMeterQueryInfo->resultInfo[i].interResultBuf); + } + + free(pMeterQueryInfo->resultInfo); + free(pMeterQueryInfo); +} + +void changeMeterQueryInfoForSuppleQuery(SMeterQueryInfo *pMeterQueryInfo, TSKEY skey, TSKEY ekey) { + if (pMeterQueryInfo == NULL) { return; } - pMeterDataInfo->pMeterQInfo = calloc(1, sizeof(SMeterQueryInfo)); - SMeterQueryInfo *pMQInfo = pMeterDataInfo->pMeterQInfo; + pMeterQueryInfo->skey = skey; + pMeterQueryInfo->ekey = ekey; + pMeterQueryInfo->lastKey = pMeterQueryInfo->skey; - pMQInfo->skey = pSupporter->rawSKey; - pMQInfo->ekey = pSupporter->rawEKey; - pMQInfo->lastKey = pSupporter->rawSKey; + pMeterQueryInfo->queryRangeSet = 0; + pMeterQueryInfo->cur.order = pMeterQueryInfo->cur.order ^ 1; + pMeterQueryInfo->cur.vnodeIndex = -1; - pMQInfo->numOfPages = 0; // one page - pMQInfo->numOfAlloc = INIT_ALLOCATE_DISK_PAGES; - pMQInfo->pageList = calloc(pMQInfo->numOfAlloc, sizeof(uint32_t)); - pMQInfo->lastResRows = 0; + // previous does not generate any results + if (pMeterQueryInfo->numOfPages == 0) { + pMeterQueryInfo->reverseFillRes = 0; + } else { + pMeterQueryInfo->reverseIndex = pMeterQueryInfo->numOfRes; + pMeterQueryInfo->reverseFillRes = 1; + } } -void incOutputPageId(SMeterQueryInfo *pMeterQInfo, uint32_t pageId) { - if (pMeterQInfo->numOfPages >= pMeterQInfo->numOfAlloc) { - pMeterQInfo->numOfAlloc = pMeterQInfo->numOfAlloc << 1; +static tFilePage *allocNewPage(SMeterQuerySupportObj *pSupporter, uint32_t *pageId) { + if (pSupporter->lastPageId == pSupporter->numOfPages - 1) { + extendDiskBuf(pSupporter, pSupporter->numOfPages + pSupporter->numOfMeters); + } - pMeterQInfo->pageList = realloc(pMeterQInfo->pageList, sizeof(uint32_t) * pMeterQInfo->numOfAlloc); + *pageId = (++pSupporter->lastPageId); + return getFilePage(pSupporter, *pageId); +} + +tFilePage *addDataPageForMeterQueryInfo(SMeterQueryInfo *pMeterQueryInfo, SMeterQuerySupportObj *pSupporter) { + uint32_t pageId = 0; + tFilePage *pPage = allocNewPage(pSupporter, &pageId); + + if (pMeterQueryInfo->numOfPages >= pMeterQueryInfo->numOfAlloc) { + pMeterQueryInfo->numOfAlloc = pMeterQueryInfo->numOfAlloc << 1; + pMeterQueryInfo->pageList = realloc(pMeterQueryInfo->pageList, sizeof(uint32_t) * pMeterQueryInfo->numOfAlloc); } - pMeterQInfo->pageList[pMeterQInfo->numOfPages++] = pageId; + pMeterQueryInfo->pageList[pMeterQueryInfo->numOfPages++] = pageId; + return pPage; } -void destroyMeterQueryInfo(SMeterQueryInfo *pMeterQInfo) { - if (pMeterQInfo == NULL) { - return; +void saveIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, SMeterQueryInfo *pMeterQueryInfo) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + pMeterQueryInfo->skey = pQuery->skey; + pMeterQueryInfo->ekey = pQuery->ekey; + pMeterQueryInfo->lastKey = pQuery->lastKey; + + assert(((pQuery->lastKey >= pQuery->skey) && QUERY_IS_ASC_QUERY(pQuery)) || + ((pQuery->lastKey <= pQuery->skey) && !QUERY_IS_ASC_QUERY(pQuery))); + + if (pRuntimeEnv->pTSBuf != NULL) { + pMeterQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); } +} + +void restoreIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, SMeterQueryInfo *pMeterQueryInfo) { + SQuery *pQuery = pRuntimeEnv->pQuery; - free(pMeterQInfo->pageList); - free(pMeterQInfo); + pQuery->skey = pMeterQueryInfo->skey; + pQuery->ekey = pMeterQueryInfo->ekey; + pQuery->lastKey = pMeterQueryInfo->lastKey; + + assert(((pQuery->lastKey >= pQuery->skey) && QUERY_IS_ASC_QUERY(pQuery)) || + ((pQuery->lastKey <= pQuery->skey) && !QUERY_IS_ASC_QUERY(pQuery))); } static void clearMeterDataBlockInfo(SMeterDataInfo *pMeterDataInfo) { @@ -5058,14 +5732,12 @@ static bool setCurrentQueryRange(SMeterDataInfo *pMeterDataInfo, SQuery *pQuery, } if (*minval > *maxval) { - qTrace("QInfo:%p vid:%d sid:%d id:%s, no result in files, qrange:%lld-%lld, lastKey:%lld", - pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pMeterQInfo->skey, pMeterQInfo->ekey, - pMeterQInfo->lastKey); + qTrace("QInfo:%p vid:%d sid:%d id:%s, no result in files, qrange:%lld-%lld, lastKey:%lld", pQInfo, pMeterObj->vnode, + pMeterObj->sid, pMeterObj->meterId, pMeterQInfo->skey, pMeterQInfo->ekey, pMeterQInfo->lastKey); return false; } else { - qTrace("QInfo:%p vid:%d sid:%d id:%s, query in files, qrange:%lld-%lld, lastKey:%lld", - pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pMeterQInfo->skey, pMeterQInfo->ekey, - pMeterQInfo->lastKey); + qTrace("QInfo:%p vid:%d sid:%d id:%s, query in files, qrange:%lld-%lld, lastKey:%lld", pQInfo, pMeterObj->vnode, + pMeterObj->sid, pMeterObj->meterId, pMeterQInfo->skey, pMeterQInfo->ekey, pMeterQInfo->lastKey); return true; } } @@ -5080,9 +5752,9 @@ static bool setCurrentQueryRange(SMeterDataInfo *pMeterDataInfo, SQuery *pQuery, */ uint32_t getDataBlocksForMeters(SMeterQuerySupportObj *pSupporter, SQuery *pQuery, char *pHeaderData, int32_t numOfMeters, SQueryFileInfo *pQueryFileInfo, SMeterDataInfo **pMeterDataInfo) { - uint32_t numOfBlocks = 0; - SQInfo * pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery); - SQueryCostStatistics *pSummary = &pSupporter->runtimeEnv.summary; + uint32_t numOfBlocks = 0; + SQInfo * pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery); + SQueryCostSummary *pSummary = &pSupporter->runtimeEnv.summary; TSKEY minval, maxval; @@ -5202,12 +5874,18 @@ static int32_t blockAccessOrderComparator(const void *pLeft, const void *pRight, int32_t createDataBlocksInfoEx(SMeterDataInfo **pMeterDataInfo, int32_t numOfMeters, SMeterDataBlockInfoEx **pDataBlockInfoEx, int32_t numOfCompBlocks, int32_t *nAllocBlocksInfoSize, int64_t addr) { - /* release allocated memory first */ + // release allocated memory first freeDataBlockFieldInfo(*pDataBlockInfoEx, *nAllocBlocksInfoSize); if (*nAllocBlocksInfoSize == 0 || *nAllocBlocksInfoSize < numOfCompBlocks) { - *pDataBlockInfoEx = - (SMeterDataBlockInfoEx *)realloc((*pDataBlockInfoEx), sizeof(SMeterDataBlockInfoEx) * numOfCompBlocks); + char *tmp = realloc((*pDataBlockInfoEx), sizeof(SMeterDataBlockInfoEx) * numOfCompBlocks); + if (tmp == NULL) { + tfree(*pDataBlockInfoEx); + return -1; + } else { + *pDataBlockInfoEx = (SMeterDataBlockInfoEx *)tmp; + } + memset((*pDataBlockInfoEx), 0, sizeof(SMeterDataBlockInfoEx) * numOfCompBlocks); *nAllocBlocksInfoSize = numOfCompBlocks; } @@ -5218,6 +5896,14 @@ int32_t createDataBlocksInfoEx(SMeterDataInfo **pMeterDataInfo, int32_t numOfMet supporter.blockIndexArray = calloc(1, sizeof(int32_t) * numOfMeters); supporter.pDataBlockInfoEx = calloc(1, POINTER_BYTES * numOfMeters); + if (supporter.numOfBlocksPerMeter == NULL || supporter.blockIndexArray == NULL || + supporter.pDataBlockInfoEx == NULL) { + tfree(supporter.numOfBlocksPerMeter); + tfree(supporter.blockIndexArray); + tfree(supporter.pDataBlockInfoEx); + return -1; + } + int32_t cnt = 0; int32_t numOfQualMeters = 0; for (int32_t j = 0; j < numOfMeters; ++j) { @@ -5227,6 +5913,8 @@ int32_t createDataBlocksInfoEx(SMeterDataInfo **pMeterDataInfo, int32_t numOfMet SCompBlock **pBlock = pMeterDataInfo[j]->pBlock; supporter.numOfBlocksPerMeter[numOfQualMeters] = pMeterDataInfo[j]->numOfBlocks; + + // TODO handle failed to allocate memory supporter.pDataBlockInfoEx[numOfQualMeters] = calloc(1, sizeof(SMeterDataBlockInfoEx) * pMeterDataInfo[j]->numOfBlocks); @@ -5297,61 +5985,75 @@ int32_t createDataBlocksInfoEx(SMeterDataInfo **pMeterDataInfo, int32_t numOfMet * @param pRuntimeEnv * @param pDataBlockInfoEx */ -void setExecutionContext(SMeterQuerySupportObj *pSupporter, SOutputRes *outputRes, int32_t meterIdx, int32_t groupIdx) { +void setExecutionContext(SMeterQuerySupportObj *pSupporter, SOutputRes *outputRes, int32_t meterIdx, int32_t groupIdx, + SMeterQueryInfo *pMeterQueryInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pSupporter->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - setOutputBuffer(pRuntimeEnv, &outputRes[groupIdx]); - - if (outputRes[groupIdx].numOfRows == 0) { - initCtxOutputBuf(pRuntimeEnv); - } - - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { - pRuntimeEnv->pCtx[j].numOfOutputElems = 0; - pRuntimeEnv->pCtx[j].numOfIteratedElems = 0; - } + setGroupOutputBuffer(pRuntimeEnv, &outputRes[groupIdx]); + initCtxOutputBuf(pRuntimeEnv); vnodeSetTagValueInParam(pSupporter->pSidSet, pRuntimeEnv, pSupporter->pMeterSidExtInfo[meterIdx]); -} -static void setOutputBuffer(SQueryRuntimeEnv *pRuntimeEnv, SOutputRes *pResult) { - // Note: pResult->result[i]->numOfElems == 0, there is only fixed number of - // results for each group - for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutputCols; ++i) { - // the value is not set yet - assert(pResult->result[i]->numOfElems == 0); - pRuntimeEnv->pCtx[i].aOutputBuf = - pResult->result[i]->data + pRuntimeEnv->pCtx[i].outputBytes * pResult->result[i]->numOfElems; + // set the right cursor position for ts buffer + if (pSupporter->runtimeEnv.pTSBuf != NULL) { + if (pMeterQueryInfo->cur.vnodeIndex == -1) { + pMeterQueryInfo->tag = pRuntimeEnv->pCtx[0].tag.i64Key; + + tsBufGetElemStartPos(pSupporter->runtimeEnv.pTSBuf, 0, pMeterQueryInfo->tag); + } else { + tsBufSetCursor(pSupporter->runtimeEnv.pTSBuf, &pMeterQueryInfo->cur); + } } } -static tFilePage *allocNewPage(SMeterQuerySupportObj *pSupporter, uint32_t *pageId) { - if (pSupporter->lastPageId == pSupporter->numOfPages - 1) { - extendDiskBuf(pSupporter, pSupporter->numOfPages + pSupporter->numOfMeters); - } +static void setGroupOutputBuffer(SQueryRuntimeEnv *pRuntimeEnv, SOutputRes *pResult) { + SQuery *pQuery = pRuntimeEnv->pQuery; - *pageId = (++pSupporter->lastPageId); + // Note: pResult->result[i]->numOfElems == 0, there is only fixed number of results for each group + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + assert(pResult->result[i]->numOfElems == 0 || pResult->result[i]->numOfElems == 1); - return getFilePage(pSupporter, *pageId); + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + pCtx->aOutputBuf = pResult->result[i]->data + pCtx->outputBytes * pResult->result[i]->numOfElems; + + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; + } + + /* + * set the output buffer information and intermediate buffer + * not all queries require the interResultBuf, such as COUNT + */ + pCtx->resultInfo = &pResult->resultInfo[i]; + + // set super table query flag + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + if (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + pResInfo->superTableQ = true; + } + } } static char *getOutputResPos(SQueryRuntimeEnv *pRuntimeEnv, tFilePage *pData, int32_t row, int32_t col) { + // the output for each record should be less than the DEFAULT_INTERN_BUF_SIZE + assert(pRuntimeEnv->pCtx[col].outputBytes <= DEFAULT_INTERN_BUF_SIZE); + return (char *)pData->data + pRuntimeEnv->offset[col] * pRuntimeEnv->numOfRowsPerPage + pRuntimeEnv->pCtx[col].outputBytes * row; } -void setCtxOutputPointerForSupplementScan(SMeterQuerySupportObj *pSupporter, SMeterQueryInfo *sqinfo) { +void setCtxOutputPointerForSupplementScan(SMeterQuerySupportObj *pSupporter, SMeterQueryInfo *pMeterQueryInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pSupporter->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - int32_t index = sqinfo->reverseIndex; + int32_t index = pMeterQueryInfo->reverseIndex; tFilePage *pData = NULL; int32_t i = 0; // find the position for this output result - for (; i < sqinfo->numOfPages; ++i) { - pData = getFilePage(pSupporter, sqinfo->pageList[i]); + for (; i < pMeterQueryInfo->numOfPages; ++i) { + pData = getFilePage(pSupporter, pMeterQueryInfo->pageList[i]); if (index <= pData->numOfElems) { break; } @@ -5369,26 +6071,33 @@ void setCtxOutputPointerForSupplementScan(SMeterQuerySupportObj *pSupporter, SMe } for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { - pRuntimeEnv->pCtx[k].aOutputBuf = getOutputResPos(pRuntimeEnv, pData, index - 1, k); - if (!pRuntimeEnv->go[k]) { + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[k]; + + pCtx->aOutputBuf = getOutputResPos(pRuntimeEnv, pData, index - 1, k); + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->complete) { continue; } int32_t functId = pQuery->pSelectExpr[k].pBase.functionId; - /* setup runtime environment */ + // setup runtime environment if ((QUERY_IS_ASC_QUERY(pQuery) && functId == TSDB_FUNC_FIRST_DST) || (!QUERY_IS_ASC_QUERY(pQuery) && functId == TSDB_FUNC_LAST_DST)) { - if (sqinfo->lastResRows == 0) { - aAggs[functId].init(&pRuntimeEnv->pCtx[k]); + if (pMeterQueryInfo->lastResRows == 0) { + pCtx->currentStage = 0; + + resetResultInfo(pResInfo); + aAggs[functId].init(pCtx); } } } + // the first column is always the timestamp for interval query TSKEY ts = *(TSKEY *)pRuntimeEnv->pCtx[0].aOutputBuf; SMeterObj *pMeterObj = pRuntimeEnv->pMeterObj; - qTrace("QInfo:%p vid:%d sid:%d id:%s, set output result pointer, ts:%lld, index:%d", - GET_QINFO_ADDR(pQuery), pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, ts, sqinfo->reverseIndex); + qTrace("QInfo:%p vid:%d sid:%d id:%s, set output result pointer, ts:%lld, index:%d", GET_QINFO_ADDR(pQuery), + pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, ts, pMeterQueryInfo->reverseIndex); } void validateTimestampForSupplementResult(SQueryRuntimeEnv *pRuntimeEnv, int64_t numOfIncrementRes) { @@ -5405,65 +6114,75 @@ void validateTimestampForSupplementResult(SQueryRuntimeEnv *pRuntimeEnv, int64_t } } -void setOutputBufferForIntervalQuery(SMeterQuerySupportObj *pSupporter, SMeterQueryInfo *sqinfo) { +void setOutputBufferForIntervalQuery(SMeterQuerySupportObj *pSupporter, SMeterQueryInfo *pMeterQueryInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pSupporter->runtimeEnv; tFilePage * pData = NULL; - uint32_t newPageId = 0; // in the first scan, new space needed for results - if (sqinfo->numOfPages == 0) { - pData = allocNewPage(pSupporter, &newPageId); - incOutputPageId(sqinfo, newPageId); + if (pMeterQueryInfo->numOfPages == 0) { + pData = addDataPageForMeterQueryInfo(pMeterQueryInfo, pSupporter); } else { - int32_t lastPageId = sqinfo->pageList[sqinfo->numOfPages - 1]; + int32_t lastPageId = pMeterQueryInfo->pageList[pMeterQueryInfo->numOfPages - 1]; pData = getFilePage(pSupporter, lastPageId); if (pData->numOfElems >= pRuntimeEnv->numOfRowsPerPage) { - pData = allocNewPage(pSupporter, &newPageId); - incOutputPageId(sqinfo, newPageId); - - // number of elements must be 0 for newly allocated buffer - assert(pData->numOfElems == 0); + pData = addDataPageForMeterQueryInfo(pMeterQueryInfo, pSupporter); + assert(pData->numOfElems == 0); // number of elements must be 0 for new allocated buffer } } for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutputCols; ++i) { pRuntimeEnv->pCtx[i].aOutputBuf = getOutputResPos(pRuntimeEnv, pData, pData->numOfElems, i); + pRuntimeEnv->pCtx[i].resultInfo = &pMeterQueryInfo->resultInfo[i]; } } -void setIntervalQueryExecutionContext(SMeterQuerySupportObj *pSupporter, int32_t meterIdx, SMeterQueryInfo *sqinfo) { +void setIntervalQueryExecutionContext(SMeterQuerySupportObj *pSupporter, int32_t meterIdx, + SMeterQueryInfo *pMeterQueryInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pSupporter->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; if (IS_MASTER_SCAN(pRuntimeEnv)) { - setOutputBufferForIntervalQuery(pSupporter, sqinfo); + setOutputBufferForIntervalQuery(pSupporter, pMeterQueryInfo); - if (sqinfo->lastResRows == 0) { + if (pMeterQueryInfo->lastResRows == 0) { initCtxOutputBuf(pRuntimeEnv); } - /* reset the number of iterated elements, once this function is called. since the pCtx for different - */ + // reset the number of iterated elements, once this function is called. since the pCtx for different for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { - pRuntimeEnv->pCtx[j].numOfOutputElems = 0; - pRuntimeEnv->pCtx[j].numOfIteratedElems = 0; + // pRuntimeEnv->pCtx[j].numOfIteratedElems = 0; } } else { - if (sqinfo->reverseFillRes) { - setCtxOutputPointerForSupplementScan(pSupporter, sqinfo); - } else { // find the correct output position of existed results during - // reverse scan data blocks - /* - * the master scan does not produce any results yet, - * new spaces needed to be allocated during supplementary scan - */ - setOutputBufferForIntervalQuery(pSupporter, sqinfo); + if (pMeterQueryInfo->reverseFillRes) { + setCtxOutputPointerForSupplementScan(pSupporter, pMeterQueryInfo); + } else { + /* + * set output buffer for reverse scan data blocks + * find the correct output position of existed results during + * + * If the master scan does not produce any results, new spaces needed to be allocated during supplement scan + */ + setOutputBufferForIntervalQuery(pSupporter, pMeterQueryInfo); } } vnodeSetTagValueInParam(pSupporter->pSidSet, pRuntimeEnv, pSupporter->pMeterSidExtInfo[meterIdx]); + + // both the master and supplement scan needs to set the correct ts comp start position + if (pSupporter->runtimeEnv.pTSBuf != NULL) { + if (pMeterQueryInfo->cur.vnodeIndex == -1) { + pMeterQueryInfo->tag = pRuntimeEnv->pCtx[0].tag.i64Key; + + tsBufGetElemStartPos(pSupporter->runtimeEnv.pTSBuf, 0, pMeterQueryInfo->tag); + + // keep the cursor info of current meter + pMeterQueryInfo->cur = pSupporter->runtimeEnv.pTSBuf->cur; + } else { + tsBufSetCursor(pSupporter->runtimeEnv.pTSBuf, &pMeterQueryInfo->cur); + } + } } static void doApplyIntervalQueryOnBlock(SMeterQuerySupportObj *pSupporter, SMeterQueryInfo *pInfo, @@ -5506,16 +6225,16 @@ static void doApplyIntervalQueryOnBlock(SMeterQuerySupportObj *pSupporter, SMete } /* - * 1. there may be more date that satisfy current query interval, other than current block, we need to - * try next data blocks + * 1. there may be more date that satisfy current query interval, other than + * current block, we need to try next data blocks * 2. query completed, since reaches the upper bound of the main query range */ if (QUERY_IS_ASC_QUERY(pQuery)) { if (pQuery->lastKey > pBlockInfo->keyLast || pQuery->lastKey > pSupporter->rawEKey || nextKey > pSupporter->rawEKey) { /* - * current interval query is completed, set query result flag closed and try next data block - * if pQuery->ekey == pSupporter->rawEKey, whole query is completed + * current interval query is completed, set query result flag closed and + * try next data block if pQuery->ekey == pSupporter->rawEKey, whole query is completed */ if (pQuery->lastKey > pBlockInfo->keyLast) { assert(pQuery->ekey >= pBlockInfo->keyLast); @@ -5526,16 +6245,14 @@ static void doApplyIntervalQueryOnBlock(SMeterQuerySupportObj *pSupporter, SMete assert(queryCompleted); saveResult(pSupporter, pInfo, pInfo->lastResRows); - /* - * save the pQuery->lastKey for retrieve data in cache, actually, there will be no qualified data in cache. - */ - saveIntervalQueryRange(pQuery, pInfo); + // save the pQuery->lastKey for retrieve data in cache, actually, there will be no qualified data in cache. + saveIntervalQueryRange(pRuntimeEnv, pInfo); } else if (pQuery->ekey == pBlockInfo->keyLast) { /* current interval query is completed, set the next query range on other data blocks if exist */ int64_t prevEKey = pQuery->ekey; getAlignedIntervalQueryRange(pQuery, pQuery->lastKey, pSupporter->rawSKey, pSupporter->rawEKey); - saveIntervalQueryRange(pQuery, pInfo); + saveIntervalQueryRange(pRuntimeEnv, pInfo); assert(queryCompleted && prevEKey < pQuery->skey); if (pInfo->lastResRows > 0) { @@ -5543,19 +6260,19 @@ static void doApplyIntervalQueryOnBlock(SMeterQuerySupportObj *pSupporter, SMete } } else { /* - * Data that satisfy current query range may locate in current block and blocks that are - * directly right next to current block. Therefore, we need to keep the query range(interval) - * unchanged until reaching the direct next data block, while only forwards the pQuery->lastKey. + * Data that satisfy current query range may locate in current block and blocks that are directly right + * next to current block. Therefore, we need to keep the query range(interval) unchanged until reaching + * the direct next data block, while only forwards the pQuery->lastKey. * * With the information of the directly next data block, whether locates in cache or disk, * current interval query being completed or not can be decided. */ - saveIntervalQueryRange(pQuery, pInfo); + saveIntervalQueryRange(pRuntimeEnv, pInfo); assert(pQuery->lastKey > pBlockInfo->keyLast && pQuery->lastKey <= pQuery->ekey); /* - * if current block is the last block of current file, - * we still close the result flag, and merge with other meters in the same group + * if current block is the last block of current file, we still close the result flag, and + * merge with other meters in the same group */ if (queryCompleted) { saveResult(pSupporter, pInfo, pInfo->lastResRows); @@ -5577,15 +6294,16 @@ static void doApplyIntervalQueryOnBlock(SMeterQuerySupportObj *pSupporter, SMete saveResult(pSupporter, pInfo, pInfo->lastResRows); /* - * save the pQuery->lastKey for retrieve data in cache, actually, there will be no qualified data in cache. + * save the pQuery->lastKey for retrieve data in cache, actually, + * there will be no qualified data in cache. */ - saveIntervalQueryRange(pQuery, pInfo); + saveIntervalQueryRange(pRuntimeEnv, pInfo); } else if (pQuery->ekey == pBlockInfo->keyFirst) { - /* current interval query is completed, set the next query range on other data blocks if exist */ + // current interval query is completed, set the next query range on other data blocks if exist int64_t prevEKey = pQuery->ekey; getAlignedIntervalQueryRange(pQuery, pQuery->lastKey, pSupporter->rawSKey, pSupporter->rawEKey); - saveIntervalQueryRange(pQuery, pInfo); + saveIntervalQueryRange(pRuntimeEnv, pInfo); assert(queryCompleted && prevEKey > pQuery->skey); if (pInfo->lastResRows > 0) { @@ -5600,12 +6318,12 @@ static void doApplyIntervalQueryOnBlock(SMeterQuerySupportObj *pSupporter, SMete * With the information of the directly next data block, whether locates in cache or disk, * current interval query being completed or not can be decided. */ - saveIntervalQueryRange(pQuery, pInfo); + saveIntervalQueryRange(pRuntimeEnv, pInfo); assert(pQuery->lastKey < pBlockInfo->keyFirst && pQuery->lastKey >= pQuery->ekey); /* - * if current block is the last block of current file, - * we still close the result flag, and merge with other meters in the same group + * if current block is the last block of current file, we still close the result + * flag, and merge with other meters in the same group */ if (queryCompleted) { saveResult(pSupporter, pInfo, pInfo->lastResRows); @@ -5624,7 +6342,7 @@ static void doApplyIntervalQueryOnBlock(SMeterQuerySupportObj *pSupporter, SMete /* still in the same block to query */ getAlignedIntervalQueryRange(pQuery, nextKey, pSupporter->rawSKey, pSupporter->rawEKey); - saveIntervalQueryRange(pQuery, pInfo); + saveIntervalQueryRange(pRuntimeEnv, pInfo); int32_t newPos = searchFn((char *)pPrimaryCol, pBlockInfo->size, pQuery->skey, pQuery->order.order); assert(newPos == pQuery->pos + steps * factor); @@ -5648,11 +6366,19 @@ int64_t getNextAccessedKeyInData(SQuery *pQuery, int64_t *pPrimaryCol, SBlockInf return key; } -void setIntervalQueryRange(SMeterQuerySupportObj *pSupporter, TSKEY key, SMeterDataInfo *pInfoEx) { +/* + * There are two cases to handle: + * + * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including pQuery->lastKey, + * pQuery->skey, and pQuery->eKey. + * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be + * merged during merge stage. In this case, we need the pMeterQueryInfo->lastResRows to decide if there + * is a previous result generated or not. + */ +void setIntervalQueryRange(SMeterQueryInfo *pMeterQueryInfo, SMeterQuerySupportObj *pSupporter, TSKEY key) { SQueryRuntimeEnv *pRuntimeEnv = &pSupporter->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - SMeterQueryInfo *pMeterQueryInfo = pInfoEx->pMeterQInfo; if (pMeterQueryInfo->queryRangeSet) { assert((QUERY_IS_ASC_QUERY(pQuery) && pQuery->lastKey >= pQuery->skey) || (!QUERY_IS_ASC_QUERY(pQuery) && pQuery->lastKey <= pQuery->skey)); @@ -5663,9 +6389,9 @@ void setIntervalQueryRange(SMeterQuerySupportObj *pSupporter, TSKEY key, SMeterD * otherwise, keep the previous query range and proceed */ getAlignedIntervalQueryRange(pQuery, key, pSupporter->rawSKey, pSupporter->rawEKey); - saveIntervalQueryRange(pQuery, pMeterQueryInfo); + saveIntervalQueryRange(pRuntimeEnv, pMeterQueryInfo); - // previous query does not be closed, close it + // previous query does not be closed, save the results and close it if (pMeterQueryInfo->lastResRows > 0) { saveResult(pSupporter, pMeterQueryInfo, pMeterQueryInfo->lastResRows); } @@ -5673,13 +6399,6 @@ void setIntervalQueryRange(SMeterQuerySupportObj *pSupporter, TSKEY key, SMeterD /* current query not completed, continue. do nothing with respect to query range, */ } } else { - /* - * There are two cases to handle for the first block. - * 1. Query range is not set yet. we need to set the query range, pQuery->lastKey, pQuery->skey, pQuery->eKey. - * 2. Query range is set and in progress. There may be another result with the same query ranges to be merged - * during merge stage. However, in this case, we need the - * pMeterQueryInfo->lastQueryClosed to decide if there is a previous result be generated or not. - */ pQuery->skey = key; assert(pMeterQueryInfo->lastResRows == 0); @@ -5690,7 +6409,7 @@ void setIntervalQueryRange(SMeterQuerySupportObj *pSupporter, TSKEY key, SMeterD } getAlignedIntervalQueryRange(pQuery, pQuery->skey, pSupporter->rawSKey, pSupporter->rawEKey); - saveIntervalQueryRange(pQuery, pMeterQueryInfo); + saveIntervalQueryRange(pRuntimeEnv, pMeterQueryInfo); pMeterQueryInfo->queryRangeSet = 1; } } @@ -5710,9 +6429,13 @@ static void setTimestampRange(SQueryRuntimeEnv *pRuntimeEnv, int64_t stime, int6 for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; - if (functionId == TSDB_FUNC_SPREAD || functionId == TSDB_FUNC_SPREAD_DST) { - pRuntimeEnv->pCtx[i].intermediateBuf[1].dKey = stime; - pRuntimeEnv->pCtx[i].intermediateBuf[2].dKey = etime; + + if (functionId == TSDB_FUNC_SPREAD) { + pRuntimeEnv->pCtx[i].param[1].dKey = stime; + pRuntimeEnv->pCtx[i].param[2].dKey = etime; + + pRuntimeEnv->pCtx[i].param[1].nType = TSDB_DATA_TYPE_DOUBLE; + pRuntimeEnv->pCtx[i].param[2].nType = TSDB_DATA_TYPE_DOUBLE; } } } @@ -5755,12 +6478,16 @@ int32_t LoadDatablockOnDemand(SCompBlock *pBlock, SField **pFields, int8_t *blkS req |= aAggs[functID].dataReqFunc(&pRuntimeEnv->pCtx[i], pBlock->keyFirst, pBlock->keyLast, pQuery->pSelectExpr[i].pBase.colInfo.colId, *blkStatus); } + + if (pRuntimeEnv->pTSBuf > 0) { + req |= BLK_DATA_ALL_NEEDED; + } } if (req == BLK_DATA_NO_NEEDED) { qTrace("QInfo:%p vid:%d sid:%d id:%s, slot:%d, data block ignored, brange:%lld-%lld, rows:%d", - GET_QINFO_ADDR(pQuery), pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->slot, pBlock->keyFirst, - pBlock->keyLast, pBlock->numOfPoints); + GET_QINFO_ADDR(pQuery), pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->slot, + pBlock->keyFirst, pBlock->keyLast, pBlock->numOfPoints); setTimestampRange(pRuntimeEnv, pBlock->keyFirst, pBlock->keyLast); } else if (req == BLK_DATA_FILEDS_NEEDED) { @@ -5773,7 +6500,6 @@ int32_t LoadDatablockOnDemand(SCompBlock *pBlock, SField **pFields, int8_t *blkS } } else { _load_all: - // load sfield first if (loadDataBlockFieldsInfo(pRuntimeEnv, pQueryFileInfo, pBlock, pFields) < 0) { return DISK_DATA_LOAD_FAILED; } @@ -5791,8 +6517,8 @@ int32_t LoadDatablockOnDemand(SCompBlock *pBlock, SField **pFields, int8_t *blkS pQuery->slot); #endif qTrace("QInfo:%p id:%s slot:%d, data block ignored by pre-filter, fields loaded, brange:%lld-%lld, rows:%d", - GET_QINFO_ADDR(pQuery), pMeterObj->meterId, pQuery->slot, pBlock->keyFirst, pBlock->keyLast, - pBlock->numOfPoints); + GET_QINFO_ADDR(pQuery), pMeterObj->meterId, pQuery->slot, pBlock->keyFirst, pBlock->keyLast, + pBlock->numOfPoints); return DISK_DATA_DISCARDED; } } @@ -5814,6 +6540,7 @@ int32_t LoadDatablockOnDemand(SCompBlock *pBlock, SField **pFields, int8_t *blkS /* find first qualified record position in this block */ if (loadTS) { + /* find first qualified record position in this block */ pQuery->pos = searchFn(pRuntimeEnv->primaryColBuffer->data, pBlock->numOfPoints, pQuery->lastKey, pQuery->order.order); /* boundary timestamp check */ @@ -5838,29 +6565,11 @@ bool onDemandLoadDatablock(SQuery *pQuery, int16_t queryRangeSet) { return (pQuery->nAggTimeInterval == 0) || ((queryRangeSet == 1) && (pQuery->nAggTimeInterval > 0)); } -void saveIntervalQueryRange(SQuery *pQuery, SMeterQueryInfo *pInfo) { - pInfo->skey = pQuery->skey; - pInfo->ekey = pQuery->ekey; - pInfo->lastKey = pQuery->lastKey; - - assert(((pQuery->lastKey >= pQuery->skey) && QUERY_IS_ASC_QUERY(pQuery)) || - ((pQuery->lastKey <= pQuery->skey) && !QUERY_IS_ASC_QUERY(pQuery))); -} - -void restoreIntervalQueryRange(SQuery *pQuery, SMeterQueryInfo *pInfo) { - pQuery->skey = pInfo->skey; - pQuery->ekey = pInfo->ekey; - pQuery->lastKey = pInfo->lastKey; - - assert(((pQuery->lastKey >= pQuery->skey) && QUERY_IS_ASC_QUERY(pQuery)) || - ((pQuery->lastKey <= pQuery->skey) && !QUERY_IS_ASC_QUERY(pQuery))); -} - -static void validateResultBuf(SMeterQuerySupportObj *pSupporter, SMeterQueryInfo *sqinfo) { +static void validateResultBuf(SMeterQuerySupportObj *pSupporter, SMeterQueryInfo *pMeterQueryInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pSupporter->runtimeEnv; SQuery * pQuery = pSupporter->runtimeEnv.pQuery; - tFilePage *newOutput = getFilePage(pSupporter, sqinfo->pageList[sqinfo->numOfPages - 1]); + tFilePage *newOutput = getFilePage(pSupporter, pMeterQueryInfo->pageList[pMeterQueryInfo->numOfPages - 1]); for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { assert(pRuntimeEnv->pCtx[i].aOutputBuf - newOutput->data < DEFAULT_INTERN_BUF_SIZE); } @@ -5870,6 +6579,7 @@ void saveResult(SMeterQuerySupportObj *pSupporter, SMeterQueryInfo *pMeterQueryI SQueryRuntimeEnv *pRuntimeEnv = &pSupporter->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; + // no results generated, do nothing for master scan if (numOfResult <= 0) { if (IS_MASTER_SCAN(pRuntimeEnv)) { return; @@ -5888,10 +6598,9 @@ void saveResult(SMeterQuerySupportObj *pSupporter, SMeterQueryInfo *pMeterQueryI * there is no results during the supplementary scan. */ TSKEY ts = *(TSKEY *)pRuntimeEnv->pCtx[0].aOutputBuf; - if (ts == pRuntimeEnv->pCtx[0].nStartQueryTimestamp && pMeterQueryInfo->reverseIndex > 0) { assert(pMeterQueryInfo->numOfRes >= 0 && pMeterQueryInfo->reverseIndex > 0 && - pMeterQueryInfo->reverseIndex <= pMeterQueryInfo->numOfRes); + pMeterQueryInfo->reverseIndex <= pMeterQueryInfo->numOfRes); // backward one step from the previous position, the start position is (pMeterQueryInfo->numOfRows-1); pMeterQueryInfo->reverseIndex -= 1; @@ -5908,7 +6617,7 @@ void saveResult(SMeterQuerySupportObj *pSupporter, SMeterQueryInfo *pMeterQueryI if (IS_SUPPLEMENT_SCAN(pRuntimeEnv) && pMeterQueryInfo->reverseFillRes == 1) { assert(pMeterQueryInfo->numOfRes > 0 && pMeterQueryInfo->reverseIndex > 0 && - pMeterQueryInfo->reverseIndex <= pMeterQueryInfo->numOfRes); + pMeterQueryInfo->reverseIndex <= pMeterQueryInfo->numOfRes); // backward one step from the previous position, the start position is (pMeterQueryInfo->numOfRows-1); pMeterQueryInfo->reverseIndex -= 1; setCtxOutputPointerForSupplementScan(pSupporter, pMeterQueryInfo); @@ -5916,10 +6625,8 @@ void saveResult(SMeterQuerySupportObj *pSupporter, SMeterQueryInfo *pMeterQueryI int32_t pageId = pMeterQueryInfo->pageList[pMeterQueryInfo->numOfPages - 1]; tFilePage *pData = getFilePage(pSupporter, pageId); - // in handling records occuring around '1970-01-01', the aligned start - // timestamp may be 0. + // in handling records occuring around '1970-01-01', the aligned start timestamp may be 0. TSKEY ts = *(TSKEY *)getOutputResPos(pRuntimeEnv, pData, pData->numOfElems, 0); - assert(ts >= 0); SMeterObj *pMeterObj = pRuntimeEnv->pMeterObj; qTrace("QInfo:%p vid:%d sid:%d id:%s, save results, ts:%lld, total:%d", GET_QINFO_ADDR(pQuery), pMeterObj->vnode, @@ -5931,10 +6638,14 @@ void saveResult(SMeterQuerySupportObj *pSupporter, SMeterQueryInfo *pMeterQueryI setOutputBufferForIntervalQuery(pSupporter, pMeterQueryInfo); + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + resetResultInfo(&pMeterQueryInfo->resultInfo[i]); + } + validateResultBuf(pSupporter, pMeterQueryInfo); initCtxOutputBuf(pRuntimeEnv); #if 0 - SSchema sc[TSDB_MAX_COLUMNS] = {0}; + SSchema sc[TSDB_MAX_COLUMNS] = {0}; sc[0].type = TSDB_DATA_TYPE_BIGINT; sc[0].bytes = 8; @@ -5950,33 +6661,46 @@ void saveResult(SMeterQuerySupportObj *pSupporter, SMeterQueryInfo *pMeterQueryI } } +static int32_t getSubsetNumber(SMeterQuerySupportObj *pSupporter) { + SQuery *pQuery = pSupporter->runtimeEnv.pQuery; + + int32_t totalSubset = 0; + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + totalSubset = pSupporter->runtimeEnv.usedIndex; + } else { + totalSubset = pSupporter->pSidSet->numOfSubSet; + } + + return totalSubset; +} + static int32_t doCopyFromGroupBuf(SMeterQuerySupportObj *pSupporter, SOutputRes *result, int32_t orderType) { SQueryRuntimeEnv *pRuntimeEnv = &pSupporter->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; int32_t numOfResult = 0; - /* pointsToRead is the max number of rows of results for output*/ - int32_t startIdx = 0; int32_t forward = 1; dTrace("QInfo:%p start to copy data to dest buf", GET_QINFO_ADDR(pSupporter->runtimeEnv.pQuery)); + int32_t totalSubset = getSubsetNumber(pSupporter); + if (orderType == TSQL_SO_ASC) { startIdx = pSupporter->subgroupIdx; } else { // desc - startIdx = pSupporter->pSidSet->numOfSubSet - pSupporter->subgroupIdx - 1; + startIdx = totalSubset - pSupporter->subgroupIdx - 1; forward = -1; } - for (int32_t i = startIdx; (i < pSupporter->pSidSet->numOfSubSet) && (i >= 0); i += forward) { + for (int32_t i = startIdx; (i < totalSubset) && (i >= 0); i += forward) { if (result[i].numOfRows == 0) { pSupporter->offset = 0; pSupporter->subgroupIdx += 1; continue; } - assert(result[i].numOfRows <= 1 && pSupporter->offset <= 1); + assert(result[i].numOfRows >= 0 && pSupporter->offset <= 1); tFilePage **srcBuf = result[i].result; @@ -5984,8 +6708,7 @@ static int32_t doCopyFromGroupBuf(SMeterQuerySupportObj *pSupporter, SOutputRes int32_t oldOffset = pSupporter->offset; if (numOfRowsToCopy > pQuery->pointsToRead - numOfResult) { - /* current output space is not enough for the keep the data of this group - */ + // current output space is not enough for the keep the data of this group numOfRowsToCopy = pQuery->pointsToRead - numOfResult; pSupporter->offset += numOfRowsToCopy; } else { @@ -6029,11 +6752,6 @@ void copyFromGroupBuf(SQInfo *pQInfo, SOutputRes *result) { pQuery->pointsRead += numOfResult; assert(pQuery->pointsRead <= pQuery->pointsToRead); - - if (pSupporter->subgroupIdx == pSupporter->pSidSet->numOfSubSet) { - /* set the end of query flag */ - pSupporter->meterIdx = pSupporter->numOfMeters; - } } // todo refactor according to its called env!! @@ -6057,7 +6775,8 @@ static void applyIntervalQueryOnBlock(SMeterQuerySupportObj *pSupporter, SMeterD * or not is based on the start key of current block. */ TSKEY key = getNextAccessedKeyInData(pQuery, pPrimaryData, pBlockInfo, blockStatus); - setIntervalQueryRange(pSupporter, key, pInfoEx); + setIntervalQueryRange(pInfoEx->pMeterQInfo, pSupporter, key); + if (((pQuery->skey > pQuery->ekey) && QUERY_IS_ASC_QUERY(pQuery)) || ((pQuery->skey < pQuery->ekey) && !QUERY_IS_ASC_QUERY(pQuery))) { return; @@ -6070,22 +6789,41 @@ static void applyIntervalQueryOnBlock(SMeterQuerySupportObj *pSupporter, SMeterD int32_t steps = applyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, pPrimaryData, data, pFields, searchFn, &numOfRes); assert(numOfRes <= 1 && numOfRes >= 0 && steps > 0); - if (pInfoEx->pMeterQInfo->lastResRows == 0) { - pInfoEx->pMeterQInfo->lastResRows = numOfRes; + if (pInfo->lastResRows == 0) { + pInfo->lastResRows = numOfRes; } else { - assert(pInfoEx->pMeterQInfo->lastResRows == 1); + assert(pInfo->lastResRows == 1); } - saveIntervalQueryRange(pQuery, pInfo); + saveIntervalQueryRange(pRuntimeEnv, pInfo); } else { - doApplyIntervalQueryOnBlock(pSupporter, pInfoEx->pMeterQInfo, pBlockInfo, pPrimaryData, data, pFields, searchFn); + doApplyIntervalQueryOnBlock(pSupporter, pInfo, pBlockInfo, pPrimaryData, data, pFields, searchFn); } } -// we need to split the result into different packages. +// we need to split the refstatsult into different packages. int32_t vnodeGetResultSize(void *thandle, int32_t *numOfRows) { SQInfo *pQInfo = (SQInfo *)thandle; - return pQInfo->query.rowSize * (*numOfRows); + SQuery *pQuery = &pQInfo->query; + + /* + * get the file size and set the numOfRows to be the file size, since for tsComp query, + * the returned row size is equalled to 1 + * + * TODO handle the case that the file is too large to send back one time + */ + if (pQInfo->pMeterQuerySupporter != NULL && isTSCompQuery(pQuery) && (*numOfRows) > 0) { + struct stat fstat; + if (stat(pQuery->sdata[0]->data, &fstat) == 0) { + *numOfRows = fstat.st_size; + return fstat.st_size; + } else { + dError("QInfo:%p failed to get file info, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); + return 0; + } + } else { + return pQInfo->query.rowSize * (*numOfRows); + } } int64_t vnodeGetOffsetVal(void *thandle) { @@ -6118,8 +6856,8 @@ bool vnodeHasRemainResults(void *handle) { return false; } + // query has completed if (Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { - /* query has completed */ TSKEY ekey = taosGetRevisedEndKey(pSupporter->rawEKey, pQuery->order.order, pQuery->nAggTimeInterval, pQuery->intervalTimeUnit, pQuery->precision); int32_t numOfTotal = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY *)pRuntimeEnv->pInterpoBuf[0]->data, @@ -6165,31 +6903,76 @@ static int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, tFilePage **p return numOfRes; } +static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char* data, int32_t* size) { + SMeterObj* pObj = pQInfo->pObj; + SQuery* pQuery = &pQInfo->query; + + int tnumOfRows = vnodeList[pObj->vnode].cfg.rowsInFileBlock; + int32_t dataSize = pQInfo->query.rowSize * numOfRows; + + if (dataSize >= tsCompressMsgSize && tsCompressMsgSize > 0) { + char* compBuf = malloc((size_t) dataSize); + + // for metric query, bufIndex always be 0. + char* d = compBuf; + for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { // pQInfo->bufIndex == 0 + int32_t bytes = pQuery->pSelectExpr[col].resBytes; + + memmove(d, pQuery->sdata[col]->data + bytes * tnumOfRows * pQInfo->bufIndex, bytes * numOfRows); + d += bytes * numOfRows; + } + + *size = tsCompressString(compBuf, dataSize, 1, data, dataSize + EXTRA_BYTES, 0, 0, 0); + + dTrace("QInfo:%p compress rsp msg, before:%d, after:%d", pQInfo, dataSize, *size); + free(compBuf); + } else { // for metric query, bufIndex always be 0. + for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { // pQInfo->bufIndex == 0 + int32_t bytes = pQuery->pSelectExpr[col].resBytes; + + memmove(data, pQuery->sdata[col]->data + bytes * tnumOfRows * pQInfo->bufIndex, bytes * numOfRows); + data += bytes * numOfRows; + } + } +} + /** + * Copy the result data/file to output message buffer. + * If the result is in file format, read file from disk and copy to output buffer, compression is not involved since + * data in file is already compressed. + * In case of other result in buffer, compress the result before copy once the tsComressMsg is set. * * @param handle * @param data * @param numOfRows the number of rows that are not returned in current retrieve * @return */ -int32_t vnodeCopyQueryResultToMsg(void *handle, char *data, int32_t numOfRows) { +int32_t vnodeCopyQueryResultToMsg(void *handle, char *data, int32_t numOfRows, int32_t* size) { SQInfo *pQInfo = (SQInfo *)handle; - - SMeterObj *pObj = pQInfo->pObj; SQuery * pQuery = &pQInfo->query; assert(pQuery->pSelectExpr != NULL && pQuery->numOfOutputCols > 0); - // copy data into dst buffer directly - int tnumOfRows = vnodeList[pObj->vnode].cfg.rowsInFileBlock; - char *pData = data; + // load data from file to msg buffer + if (isTSCompQuery(pQuery)) { + int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); + + // make sure file exist + if (VALIDFD(fd)) { + size_t s = lseek(fd, 0, SEEK_END); + dTrace("QInfo:%p ts comp data return, file:%s, size:%ld", pQInfo, pQuery->sdata[0]->data, size); - /* for metric query, bufIndex always be 0. */ - for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { // pQInfo->bufIndex == 0 - int32_t bytes = pQuery->pSelectExpr[col].resBytes; + lseek(fd, 0, SEEK_SET); + read(fd, data, s); + close(fd); - memmove(pData, pQuery->sdata[col]->data + bytes * tnumOfRows * pQInfo->bufIndex, bytes * numOfRows); - pData += bytes * numOfRows; + unlink(pQuery->sdata[0]->data); + } else { + dError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, + pQuery->sdata[0]->data, strerror(errno)); + } + } else { + doCopyQueryResultToMsg(pQInfo, numOfRows, data, size); } return numOfRows; @@ -6253,7 +7036,7 @@ void vnodePrintQueryStatistics(SMeterQuerySupportObj *pSupporter) { SQuery *pQuery = pRuntimeEnv->pQuery; SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery); - SQueryCostStatistics *pSummary = &pRuntimeEnv->summary; + SQueryCostSummary *pSummary = &pRuntimeEnv->summary; pSummary->tmpBufferInDisk = pSupporter->bufSize; dTrace("QInfo:%p statis: comp blocks:%d, size:%d Bytes, elapsed time:%.2f ms", pQInfo, pSummary->readCompInfo, @@ -6263,7 +7046,8 @@ void vnodePrintQueryStatistics(SMeterQuerySupportObj *pSupporter) { pSummary->readField, pSummary->totalFieldSize, (double)pSummary->totalFieldSize / pSummary->readField, pSummary->loadFieldUs / 1000.0); - dTrace("QInfo:%p statis: file blocks:%d, size:%d Bytes, elapsed time:%.2f ms, skipped:%d, in-memory gen null:%d Bytes", + dTrace( + "QInfo:%p statis: file blocks:%d, size:%d Bytes, elapsed time:%.2f ms, skipped:%d, in-memory gen null:%d Bytes", pQInfo, pSummary->readDiskBlocks, pSummary->totalBlockSize, pSummary->loadBlocksUs / 1000.0, pSummary->skippedFileBlocks, pSummary->totalGenData); @@ -6280,9 +7064,10 @@ void vnodePrintQueryStatistics(SMeterQuerySupportObj *pSupporter) { // todo add the intermediate result save cost!! double computing = total - io; - dTrace("QInfo:%p statis: total elapsed time:%.2f ms, file:%.2f ms(%.2f%), cache:%.2f ms(%.2f%). io:%.2f ms(%.2f%)," - "comput:%.2fms(%.2f%)", - pQInfo, total / 1000.0, pSummary->fileTimeUs / 1000.0, pSummary->fileTimeUs * 100 / total, - pSummary->cacheTimeUs / 1000.0, pSummary->cacheTimeUs * 100 / total, io / 1000.0, io * 100 / total, - computing / 1000.0, computing * 100 / total); + dTrace( + "QInfo:%p statis: total elapsed time:%.2f ms, file:%.2f ms(%.2f%), cache:%.2f ms(%.2f%). io:%.2f ms(%.2f%)," + "comput:%.2fms(%.2f%)", + pQInfo, total / 1000.0, pSummary->fileTimeUs / 1000.0, pSummary->fileTimeUs * 100 / total, + pSummary->cacheTimeUs / 1000.0, pSummary->cacheTimeUs * 100 / total, io / 1000.0, io * 100 / total, + computing / 1000.0, computing * 100 / total); } diff --git a/src/system/src/vnodeQueryProcess.c b/src/system/detail/src/vnodeQueryProcess.c similarity index 79% rename from src/system/src/vnodeQueryProcess.c rename to src/system/detail/src/vnodeQueryProcess.c index f00f481f6d..73d97f5e6e 100644 --- a/src/system/src/vnodeQueryProcess.c +++ b/src/system/detail/src/vnodeQueryProcess.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include #include @@ -20,6 +21,7 @@ #include "taosmsg.h" #include "textbuffer.h" +#include "tscJoinProcess.h" #include "ttime.h" #include "vnode.h" #include "vnodeRead.h" @@ -32,6 +34,21 @@ #define FORWARD_CACHE_BLOCK_CHECK_SLOT(slot, step, maxblocks) (slot) = ((slot) + (step) + (maxblocks)) % (maxblocks); +static bool isGroupbyEachTable(SSqlGroupbyExpr *pGroupbyExpr, tSidSet *pSidset) { + if (pGroupbyExpr == NULL || pGroupbyExpr->numOfGroupCols == 0) { + return false; + } + + for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { + SColIndexEx *pColIndex = &pGroupbyExpr->columnInfo[i]; + if (pColIndex->flag == TSDB_COL_TAG) { + assert(pSidset->numOfSids == pSidset->numOfSubSet); + return true; + } + } + + return false; +} static bool doCheckWithPrevQueryRange(SQInfo *pQInfo, TSKEY nextKey, SMeterDataInfo *pMeterInfo) { SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; @@ -48,7 +65,7 @@ static bool doCheckWithPrevQueryRange(SQInfo *pQInfo, TSKEY nextKey, SMeterDataI return false; } else { // in case of interval query, forward the query range - setIntervalQueryRange(pSupporter, nextKey, pMeterInfo); + setIntervalQueryRange(pMeterInfo->pMeterQInfo, pSupporter, nextKey); } } @@ -106,15 +123,18 @@ static SMeterDataInfo *queryOnMultiDataCache(SQInfo *pQInfo, SMeterDataInfo *pMe pQInfo->pObj = pMeterObj; pRuntimeEnv->pMeterObj = pMeterObj; - setMeterQueryInfo(pSupporter, &pMeterInfo[k]); - if (pMeterInfo[k].pMeterObj == NULL) { /* no data in disk for this meter, set its pointer */ + if (pMeterInfo[k].pMeterQInfo == NULL) { + pMeterInfo[k].pMeterQInfo = createMeterQueryInfo(pQuery, pSupporter->rawSKey, pSupporter->rawEKey); + } + + if (pMeterInfo[k].pMeterObj == NULL) { // no data in disk for this meter, set its pointer setMeterDataInfo(&pMeterInfo[k], pMeterObj, k, groupIdx); } assert(pMeterInfo[k].meterOrderIdx == k && pMeterObj == pMeterInfo[k].pMeterObj); SMeterQueryInfo *pMeterQueryInfo = pMeterInfo[k].pMeterQInfo; - restoreIntervalQueryRange(pQuery, pMeterQueryInfo); + restoreIntervalQueryRange(pRuntimeEnv, pMeterQueryInfo); /* * Update the query meter column index and the corresponding filter column index @@ -129,14 +149,16 @@ static SMeterDataInfo *queryOnMultiDataCache(SQInfo *pQInfo, SMeterDataInfo *pMe if (pQuery->nAggTimeInterval == 0) { if ((pQuery->lastKey > pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) || (pQuery->lastKey < pQuery->ekey && !QUERY_IS_ASC_QUERY(pQuery))) { - dTrace("QInfo:%p vid:%d sid:%d id:%s, query completed, no need to scan data in cache. qrange:%lld-%lld, lastKey:%lld", - pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey, - pQuery->lastKey); + dTrace( + "QInfo:%p vid:%d sid:%d id:%s, query completed, no need to scan data in cache. qrange:%lld-%lld, " + "lastKey:%lld", + pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey, + pQuery->lastKey); continue; } - setExecutionContext(pSupporter, pSupporter->pResult, pMeterInfo[k].meterOrderIdx, pMeterInfo[k].groupIdx); + setExecutionContext(pSupporter, pSupporter->pResult, k, pMeterInfo[k].groupIdx, pMeterQueryInfo); } else { setIntervalQueryExecutionContext(pSupporter, k, pMeterQueryInfo); } @@ -222,8 +244,8 @@ static SMeterDataInfo *queryOnMultiDataCache(SQInfo *pQInfo, SMeterDataInfo *pMe } } - int64_t time = taosGetTimestampUs() - st; - SQueryCostStatistics *pSummary = &pRuntimeEnv->summary; + int64_t time = taosGetTimestampUs() - st; + SQueryCostSummary *pSummary = &pRuntimeEnv->summary; pSummary->blocksInCache += totalBlocks; pSummary->cacheTimeUs += time; @@ -236,11 +258,10 @@ static SMeterDataInfo *queryOnMultiDataCache(SQInfo *pQInfo, SMeterDataInfo *pMe return pMeterInfo; } -static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterQuerySupportObj *pSupporter, - SMeterDataInfo *pMeterDataInfo) { - SQuery * pQuery = &pQInfo->query; - SQueryRuntimeEnv *pRuntimeEnv = &pSupporter->runtimeEnv; - +static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo) { + SQuery * pQuery = &pQInfo->query; + SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; + SQueryRuntimeEnv * pRuntimeEnv = &pSupporter->runtimeEnv; SMeterDataBlockInfoEx *pDataBlockInfoEx = NULL; int32_t nAllocBlocksInfoSize = 0; @@ -250,9 +271,9 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterQuerySupportO int32_t vnodeId = pTempMeter->vnode; dTrace("QInfo:%p start to check data blocks in %d files", pQInfo, pRuntimeEnv->numOfFiles); - int32_t fid = QUERY_IS_ASC_QUERY(pQuery) ? -1 : INT32_MAX; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - SQueryCostStatistics *pSummary = &pRuntimeEnv->summary; + int32_t fid = QUERY_IS_ASC_QUERY(pQuery) ? -1 : INT32_MAX; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + SQueryCostSummary *pSummary = &pRuntimeEnv->summary; int64_t totalBlocks = 0; int64_t st = taosGetTimestampUs(); @@ -263,8 +284,7 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterQuerySupportO } int32_t fileIdx = vnodeGetVnodeHeaderFileIdx(&fid, pRuntimeEnv, pQuery->order.order); - if (fileIdx < 0) { - // no valid file, abort current search + if (fileIdx < 0) { // no valid file, abort current search break; } @@ -280,8 +300,15 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterQuerySupportO pQInfo, vnodeId, pQueryFileInfo, pSupporter->pSidSet, pMeterDataInfo, &numOfQualifiedMeters); dTrace("QInfo:%p file:%s, %d meters qualified", pQInfo, pQueryFileInfo->dataFilePath, numOfQualifiedMeters); - /* none of meters in query set have pHeaderData in this file, try next file - */ + if (pReqMeterDataInfo == NULL) { + dError("QInfo:%p failed to allocate memory to perform query processing, abort", pQInfo); + + pQInfo->code = TSDB_CODE_SERV_OUT_OF_MEMORY; + pQInfo->killed = 1; + return NULL; + } + + // none of meters in query set have pHeaderData in this file, try next file if (numOfQualifiedMeters == 0) { fid += step; tfree(pReqMeterDataInfo); @@ -299,8 +326,16 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterQuerySupportO continue; } - createDataBlocksInfoEx(pReqMeterDataInfo, numOfQualifiedMeters, &pDataBlockInfoEx, numOfBlocks, - &nAllocBlocksInfoSize, (int64_t)pQInfo); + int32_t n = createDataBlocksInfoEx(pReqMeterDataInfo, numOfQualifiedMeters, &pDataBlockInfoEx, numOfBlocks, + &nAllocBlocksInfoSize, (int64_t)pQInfo); + if (n < 0) { // failed to create data blocks + dError("QInfo:%p failed to allocate memory to perform query processing, abort", pQInfo); + tfree(pReqMeterDataInfo); + + pQInfo->code = TSDB_CODE_SERV_OUT_OF_MEMORY; + pQInfo->killed = 1; + return NULL; + } dTrace("QInfo:%p start to load %d blocks and check", pQInfo, numOfBlocks); int64_t TRACE_OUTPUT_BLOCK_CNT = 10000; @@ -335,20 +370,22 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterQuerySupportO pQInfo->pObj = pMeterObj; pRuntimeEnv->pMeterObj = pMeterObj; - restoreIntervalQueryRange(pQuery, pMeterQueryInfo); + restoreIntervalQueryRange(pRuntimeEnv, pMeterQueryInfo); if (pQuery->nAggTimeInterval == 0) { // normal query if ((pQuery->lastKey > pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) || (pQuery->lastKey < pQuery->ekey && !QUERY_IS_ASC_QUERY(pQuery))) { - qTrace("QInfo:%p vid:%d sid:%d id:%s, query completed, no need to scan this data block. qrange:%lld-%lld, " - "lastKey:%lld", - pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey, - pQuery->lastKey); + qTrace( + "QInfo:%p vid:%d sid:%d id:%s, query completed, no need to scan this data block. qrange:%lld-%lld, " + "lastKey:%lld", + pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey, + pQuery->lastKey); continue; } + setExecutionContext(pSupporter, pSupporter->pResult, pOneMeterDataInfo->meterOrderIdx, - pOneMeterDataInfo->groupIdx); + pOneMeterDataInfo->groupIdx, pMeterQueryInfo); } else { // interval query setIntervalQueryExecutionContext(pSupporter, pOneMeterDataInfo->meterOrderIdx, pMeterQueryInfo); } @@ -398,6 +435,7 @@ static SMeterDataInfo *queryOnMultiDataFiles(SQInfo *pQInfo, SMeterQuerySupportO setQueryStatus(pQuery, QUERY_NOT_COMPLETED); freeMeterBlockInfoEx(pDataBlockInfoEx, nAllocBlocksInfoSize); + return pMeterDataInfo; } @@ -435,6 +473,20 @@ static bool multimeterMultioutputHelper(SQInfo *pQInfo, bool *dataInDisk, bool * return false; } + if (pRuntimeEnv->pTSBuf != NULL) { + if (pRuntimeEnv->cur.vnodeIndex == -1) { + int64_t tag = pRuntimeEnv->pCtx[0].tag.i64Key; + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, tag); + + // failed to find data with the specified tag value + if (elem.vnode < 0) { + return false; + } + } else { + tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); + } + } + return true; } @@ -454,6 +506,7 @@ static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start resetMMapWindow(&pRuntimeEnv->pHeaderFiles[i]); } #endif + SPointInterpoSupporter pointInterpSupporter = {0}; pointInterpSupporterInit(pQuery, &pointInterpSupporter); @@ -464,14 +517,17 @@ static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start /* * here we set the value for before and after the specified time into the - * parameter for - * interpolation query + * parameter for interpolation query */ pointInterpSupporterSetData(pQInfo, &pointInterpSupporter); pointInterpSupporterDestroy(&pointInterpSupporter); vnodeScanAllData(pRuntimeEnv); - doFinalizeResult(pRuntimeEnv); + + // first/last_row query, do not invoke the finalize for super table query + if (!isFirstLastRowQuery(pQuery)) { + doFinalizeResult(pRuntimeEnv); + } int64_t numOfRes = getNumOfResult(pRuntimeEnv); assert(numOfRes == 1 || numOfRes == 0); @@ -497,8 +553,6 @@ static void vnodeMultiMeterMultiOutputProcessor(SQInfo *pQInfo) { SMeterObj *pOneMeter = getMeterObj(pSupporter->pMeterObj, pMeterSidExtInfo[0]->sid); resetCtxOutputBuf(pRuntimeEnv); - cleanCtxOutputBuf(pRuntimeEnv); - initCtxOutputBuf(pRuntimeEnv); if (isPointInterpoQuery(pQuery)) { assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); @@ -523,6 +577,7 @@ static void vnodeMultiMeterMultiOutputProcessor(SQInfo *pQInfo) { return; } + // get the last key of meters that belongs to this group SMeterObj *pMeterObj = getMeterObj(pSupporter->pMeterObj, pMeterSidExtInfo[k]->sid); if (pMeterObj != NULL) { if (key < pMeterObj->lastKey) { @@ -552,8 +607,8 @@ static void vnodeMultiMeterMultiOutputProcessor(SQInfo *pQInfo) { pQuery->skey = pSupporter->rawSKey; pQuery->ekey = pSupporter->rawEKey; - int64_t res = doCheckMetersInGroup(pQInfo, k, start); - if (res == 1) { + int64_t num = doCheckMetersInGroup(pQInfo, k, start); + if (num == 1) { break; } } @@ -567,14 +622,38 @@ static void vnodeMultiMeterMultiOutputProcessor(SQInfo *pQInfo) { } } } else { - int32_t start = pSids->starterPos[0]; - int32_t end = pSids->starterPos[1] - 1; + // this procedure treats all tables as single group + assert(pSupporter->meterIdx >= 0); - // NOTE: for group by interpolation query, the number of subset may be greater than 1 - assert(pSids->numOfSubSet == 1 && start == 0 && end == pSids->numOfSids - 1 && pSupporter->meterIdx >= start && - pSupporter->meterIdx <= end); + /* + * if the subgroup index is larger than 0, results generated by group by tbname,k is existed. + * we need to return it to client in the first place. + */ + if (pSupporter->subgroupIdx > 0) { + copyFromGroupBuf(pQInfo, pSupporter->pResult); + pQInfo->pointsRead += pQuery->pointsRead; - while(pSupporter->meterIdx < pSupporter->numOfMeters) { + if (pQuery->pointsRead > 0) { + return; + } + } + + if (pSupporter->meterIdx >= pSids->numOfSids) { + return; + } + + for (int32_t i = 0; i < pRuntimeEnv->usedIndex; ++i) { + SOutputRes *pOneRes = &pRuntimeEnv->pResult[i]; + clearGroupResultBuf(pOneRes, pQuery->numOfOutputCols); + } + + pRuntimeEnv->usedIndex = 0; + taosCleanUpIntHash(pRuntimeEnv->hashList); + + int32_t primeHashSlot = 10039; + pRuntimeEnv->hashList = taosInitIntHash(primeHashSlot, POINTER_BYTES, taosHashInt); + + while (pSupporter->meterIdx < pSupporter->numOfMeters) { int32_t k = pSupporter->meterIdx; if (isQueryKilled(pQuery)) { @@ -584,7 +663,7 @@ static void vnodeMultiMeterMultiOutputProcessor(SQInfo *pQInfo) { bool dataInDisk = true; bool dataInCache = true; - if (!multimeterMultioutputHelper(pQInfo, &dataInDisk, &dataInCache, k, start)) { + if (!multimeterMultioutputHelper(pQInfo, &dataInDisk, &dataInCache, k, 0)) { pQuery->skey = pSupporter->rawSKey; pQuery->ekey = pSupporter->rawEKey; @@ -597,6 +676,7 @@ static void vnodeMultiMeterMultiOutputProcessor(SQInfo *pQInfo) { resetMMapWindow(&pRuntimeEnv->pHeaderFiles[i]); } #endif + SPointInterpoSupporter pointInterpSupporter = {0}; if (normalizedFirstQueryRange(dataInDisk, dataInCache, pSupporter, &pointInterpSupporter) == false) { pQuery->skey = pSupporter->rawSKey; @@ -606,10 +686,11 @@ static void vnodeMultiMeterMultiOutputProcessor(SQInfo *pQInfo) { continue; } + // TODO handle the limit problem if (pQuery->numOfFilterCols == 0 && pQuery->limit.offset > 0) { forwardQueryStartPosition(pRuntimeEnv); - if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK|QUERY_COMPLETED)) { + if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { pQuery->skey = pSupporter->rawSKey; pQuery->ekey = pSupporter->rawEKey; @@ -622,7 +703,6 @@ static void vnodeMultiMeterMultiOutputProcessor(SQInfo *pQInfo) { vnodeUpdateFilterColumnIndex(pQuery); vnodeScanAllData(pRuntimeEnv); - doFinalizeResult(pRuntimeEnv); pQuery->pointsRead = getNumOfResult(pRuntimeEnv); doSkipResults(pRuntimeEnv); @@ -644,9 +724,12 @@ static void vnodeMultiMeterMultiOutputProcessor(SQInfo *pQInfo) { pQuery->ekey = pSupporter->rawEKey; pSupporter->meterIdx++; - if (Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)) { + // if the buffer is full or group by each table, we need to jump out of the loop + if (Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL) || + isGroupbyEachTable(pQuery->pGroupbyExpr, pSupporter->pSidSet)) { break; } + } else { // forward query range pQuery->skey = pQuery->lastKey; @@ -656,7 +739,7 @@ static void vnodeMultiMeterMultiOutputProcessor(SQInfo *pQInfo) { assert(!Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)); continue; } else { - //buffer is full, wait for the next round to retrieve data from current meter + // buffer is full, wait for the next round to retrieve data from current meter assert(Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)); break; } @@ -664,16 +747,65 @@ static void vnodeMultiMeterMultiOutputProcessor(SQInfo *pQInfo) { } } + if (!isGroupbyNormalCol(pQuery->pGroupbyExpr) && !isFirstLastRowQuery(pQuery)) { + doFinalizeResult(pRuntimeEnv); + } + + if (pRuntimeEnv->pTSBuf != NULL) { + pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur; + } + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + for (int32_t i = 0; i < pRuntimeEnv->usedIndex; ++i) { + SOutputRes *buf = &pRuntimeEnv->pResult[i]; + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + buf->numOfRows = MAX(buf->numOfRows, buf->resultInfo[j].numOfRes); + } + } + + pQInfo->pMeterQuerySupporter->subgroupIdx = 0; + pQuery->pointsRead = 0; + copyFromGroupBuf(pQInfo, pRuntimeEnv->pResult); + } + pQInfo->pointsRead += pQuery->pointsRead; pQuery->pointsOffset = pQuery->pointsToRead; moveDescOrderResultsToFront(pRuntimeEnv); - dTrace("QInfo %p vid:%d, numOfMeters:%d, index:%d, numOfGroups:%d, %d points returned, totalRead:%d totalReturn:%d," - "next skey:%lld, offset:%ld", - pQInfo, pOneMeter->vnode, pSupporter->pSidSet->numOfSids, pSupporter->meterIdx, - pSupporter->pSidSet->numOfSubSet, - pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned, pQuery->skey, pQuery->limit.offset); + dTrace( + "QInfo %p vid:%d, numOfMeters:%d, index:%d, numOfGroups:%d, %d points returned, totalRead:%d totalReturn:%d," + "next skey:%lld, offset:%lld", + pQInfo, pOneMeter->vnode, pSids->numOfSids, pSupporter->meterIdx, pSids->numOfSubSet, pQuery->pointsRead, + pQInfo->pointsRead, pQInfo->pointsReturned, pQuery->skey, pQuery->limit.offset); +} + +static void doOrderedScan(SQInfo *pQInfo) { + SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; + SQuery * pQuery = &pQInfo->query; + + if (QUERY_IS_ASC_QUERY(pQuery)) { + pSupporter->pMeterDataInfo = queryOnMultiDataFiles(pQInfo, pSupporter->pMeterDataInfo); + if (pQInfo->code != TSDB_CODE_SUCCESS) { + return; + } + + pSupporter->pMeterDataInfo = queryOnMultiDataCache(pQInfo, pSupporter->pMeterDataInfo); + } else { + pSupporter->pMeterDataInfo = queryOnMultiDataCache(pQInfo, pSupporter->pMeterDataInfo); + if (pQInfo->code != TSDB_CODE_SUCCESS) { + return; + } + + pSupporter->pMeterDataInfo = queryOnMultiDataFiles(pQInfo, pSupporter->pMeterDataInfo); + } +} + +static void setupMeterQueryInfoForSupplementQuery(SMeterQuerySupportObj *pSupporter) { + for (int32_t i = 0; i < pSupporter->numOfMeters; ++i) { + SMeterQueryInfo *pMeterQueryInfo = pSupporter->pMeterDataInfo[i].pMeterQInfo; + changeMeterQueryInfoForSuppleQuery(pMeterQueryInfo, pSupporter->rawSKey, pSupporter->rawEKey); + } } static void doMultiMeterSupplementaryScan(SQInfo *pQInfo) { @@ -688,43 +820,32 @@ static void doMultiMeterSupplementaryScan(SQInfo *pQInfo) { } SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv); - disableFunctForSuppleScanAndSetSortOrder(pRuntimeEnv, pQuery->order.order); - - SWAP(pSupporter->rawSKey, pSupporter->rawEKey, TSKEY); + disableFunctForSuppleScan(pRuntimeEnv, pQuery->order.order); - for (int32_t i = 0; i < pSupporter->numOfMeters; ++i) { - SMeterQueryInfo *pMeterQInfo = pSupporter->pMeterDataInfo[i].pMeterQInfo; - if (pMeterQInfo != NULL) { - pMeterQInfo->skey = pSupporter->rawSKey; - pMeterQInfo->ekey = pSupporter->rawEKey; - pMeterQInfo->lastKey = pMeterQInfo->skey; - pMeterQInfo->queryRangeSet = 0; - - /* previous does not generate any results*/ - if (pMeterQInfo->numOfPages == 0) { - pMeterQInfo->reverseFillRes = 0; - } else { - pMeterQInfo->reverseIndex = pMeterQInfo->numOfRes; - pMeterQInfo->reverseFillRes = 1; - } - } + if (pRuntimeEnv->pTSBuf != NULL) { + pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1; } + SWAP(pSupporter->rawSKey, pSupporter->rawEKey, TSKEY); + setupMeterQueryInfoForSupplementQuery(pSupporter); + int64_t st = taosGetTimestampMs(); - if (QUERY_IS_ASC_QUERY(pQuery)) { - pSupporter->pMeterDataInfo = queryOnMultiDataFiles(pQInfo, pSupporter, pSupporter->pMeterDataInfo); - pSupporter->pMeterDataInfo = queryOnMultiDataCache(pQInfo, pSupporter->pMeterDataInfo); - } else { - pSupporter->pMeterDataInfo = queryOnMultiDataCache(pQInfo, pSupporter->pMeterDataInfo); - pSupporter->pMeterDataInfo = queryOnMultiDataFiles(pQInfo, pSupporter, pSupporter->pMeterDataInfo); - } + doOrderedScan(pQInfo); + int64_t et = taosGetTimestampMs(); + dTrace("QInfo:%p supplementary scan completed, elapsed time: %lldms", pQInfo, et - st); + /* + * restore the env + * the meter query info is not reset to the original state + */ SWAP(pSupporter->rawSKey, pSupporter->rawEKey, TSKEY); enableFunctForMasterScan(pRuntimeEnv, pQuery->order.order); - SET_MASTER_SCAN_FLAG(pRuntimeEnv); - int64_t et = taosGetTimestampMs(); - dTrace("QInfo:%p supplementary scan completed, elapsed time: %lldms", pQInfo, et - st); + if (pRuntimeEnv->pTSBuf != NULL) { + pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1; + } + + SET_MASTER_SCAN_FLAG(pRuntimeEnv); } static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { @@ -751,6 +872,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { if (pQuery->pointsRead == 0) { vnodePrintQueryStatistics(pSupporter); } + dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); return; @@ -767,15 +889,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { dTrace("QInfo:%p main query scan start", pQInfo); int64_t st = taosGetTimestampMs(); - - if (QUERY_IS_ASC_QUERY(pQuery)) { // order: asc - pSupporter->pMeterDataInfo = queryOnMultiDataFiles(pQInfo, pSupporter, pSupporter->pMeterDataInfo); - pSupporter->pMeterDataInfo = queryOnMultiDataCache(pQInfo, pSupporter->pMeterDataInfo); - } else { // order: desc - pSupporter->pMeterDataInfo = queryOnMultiDataCache(pQInfo, pSupporter->pMeterDataInfo); - pSupporter->pMeterDataInfo = queryOnMultiDataFiles(pQInfo, pSupporter, pSupporter->pMeterDataInfo); - } - + doOrderedScan(pQInfo); int64_t et = taosGetTimestampMs(); dTrace("QInfo:%p main scan completed, elapsed time: %lldms, supplementary scan start, order:%d", pQInfo, et - st, pQuery->order.order ^ 1); @@ -801,17 +915,17 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { copyFromGroupBuf(pQInfo, pSupporter->pResult); } - /* handle the limitation of output buffer */ - // displayInterResult(pQuery->sdata, pQuery, pQuery->pointsRead); + // handle the limitation of output buffer pQInfo->pointsRead += pQuery->pointsRead; dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); } /* - * in each query, this function will be called only once, no retry for further result is not needed. + * in each query, this function will be called only once, no retry for further result. * - * select count(*)/top(field,k)/avg(field name) from table_name [where ts>now-1a] + * select count(*)/top(field,k)/avg(field name) from table_name [where ts>now-1a]; + * select count(*) from table_name group by status_column; */ static void vnodeSingleMeterFixedOutputProcessor(SQInfo *pQInfo) { SQuery * pQuery = &pQInfo->query; @@ -836,6 +950,12 @@ static void vnodeSingleMeterFixedOutputProcessor(SQInfo *pQInfo) { assert(isTopBottomQuery(pQuery)); } + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + pQInfo->pMeterQuerySupporter->subgroupIdx = 0; + pQuery->pointsRead = 0; + copyFromGroupBuf(pQInfo, pRuntimeEnv->pResult); + } + doSkipResults(pRuntimeEnv); doRevisedResultsByLimit(pQInfo); moveDescOrderResultsToFront(pRuntimeEnv); @@ -849,6 +969,11 @@ static void vnodeSingleMeterMultiOutputProcessor(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->pMeterQuerySupporter->runtimeEnv; + // for ts_comp query, re-initialized is not allowed + if (!isTSCompQuery(pQuery)) { + resetCtxOutputBuf(pRuntimeEnv); + } + while (1) { vnodeScanAllData(pRuntimeEnv); doFinalizeResult(pRuntimeEnv); @@ -877,7 +1002,6 @@ static void vnodeSingleMeterMultiOutputProcessor(SQInfo *pQInfo) { pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->limit.offset, pQuery->lastKey, pQuery->ekey); resetCtxOutputBuf(pRuntimeEnv); - cleanCtxOutputBuf(pRuntimeEnv); } doRevisedResultsByLimit(pQInfo); @@ -896,10 +1020,10 @@ static void vnodeSingleMeterMultiOutputProcessor(SQInfo *pQInfo) { dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d", pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); - resetCtxOutputBuf(pRuntimeEnv); - pQuery->pointsOffset = pQuery->pointsToRead; // restore the available buffer - assert(pQuery->pointsRead <= pQuery->pointsToRead); + if (!isTSCompQuery(pQuery)) { + assert(pQuery->pointsRead <= pQuery->pointsToRead); + } } static void vnodeSingleMeterIntervalMainLooper(SMeterQuerySupportObj *pSupporter, SQueryRuntimeEnv *pRuntimeEnv) { @@ -910,6 +1034,7 @@ static void vnodeSingleMeterIntervalMainLooper(SMeterQuerySupportObj *pSupporter (pQuery->skey >= pQuery->ekey && !QUERY_IS_ASC_QUERY(pQuery))); initCtxOutputBuf(pRuntimeEnv); + vnodeScanAllData(pRuntimeEnv); if (isQueryKilled(pQuery)) { return; @@ -923,9 +1048,9 @@ static void vnodeSingleMeterIntervalMainLooper(SMeterQuerySupportObj *pSupporter int64_t maxOutput = getNumOfResult(pRuntimeEnv); - /* here we can ignore the records in case of no interpolation */ - if (pQuery->numOfFilterCols > 0 && pQuery->limit.offset > 0 && pQuery->interpoType == TSDB_INTERPO_NONE) { - /* maxOutput <= 0, means current query does not generate any results */ + // here we can ignore the records in case of no interpolation + if ((pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) && pQuery->limit.offset > 0 && + pQuery->interpoType == TSDB_INTERPO_NONE) { // maxOutput <= 0, means current query does not generate any results // todo handle offset, in case of top/bottom interval query if (maxOutput > 0) { pQuery->limit.offset--; @@ -1040,13 +1165,12 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { pMeterObj->meterId, pMeterObj->numOfQueries, pQInfo); SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->pMeterQuerySupporter->runtimeEnv; - assert(pQuery->pGroupbyExpr == NULL && pRuntimeEnv->pMeterObj == pMeterObj); + assert(pRuntimeEnv->pMeterObj == pMeterObj); if (vnodeHasRemainResults(pQInfo)) { /* - * there are remain results that are not returned due to result - * interpolation - * So, we do keep in this procedure instead of launching retrieve procedure + * There are remain results that are not returned due to result interpolation + * So, we do keep in this procedure instead of launching retrieve procedure for next results. */ int32_t numOfInterpo = 0; @@ -1060,9 +1184,11 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { pQInfo->pointsInterpo += numOfInterpo; pQInfo->pointsRead += pQuery->pointsRead; - dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned %d points interpo, totalRead:%d totalInterpo:%d totalReturn:%d", - pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, numOfInterpo, - pQInfo->pointsRead, pQInfo->pointsInterpo, pQInfo->pointsReturned); + dTrace( + "QInfo:%p vid:%d sid:%d id:%s, %d points returned %d points interpo, totalRead:%d totalInterpo:%d " + "totalReturn:%d", + pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, numOfInterpo, + pQInfo->pointsRead, pQInfo->pointsInterpo, pQInfo->pointsReturned); dTrace("QInfo:%p reset signature", pQInfo); @@ -1072,8 +1198,29 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { return; } - /* here we have scan all qualified data in both data file and cache. */ + // here we have scan all qualified data in both data file and cache if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { + // continue to get push data from the group result + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + pQuery->pointsRead = 0; + if (pQInfo->pMeterQuerySupporter->subgroupIdx > 0) { + copyFromGroupBuf(pQInfo, pQInfo->pMeterQuerySupporter->pResult); + pQInfo->pointsRead += pQuery->pointsRead; + + if (pQuery->pointsRead > 0) { + dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned %d from group results, totalRead:%d totalReturn:%d", + pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, + pQInfo->pointsInterpo, pQInfo->pointsReturned); + + dTrace("QInfo:%p reset signature", pQInfo); + + TSDB_QINFO_RESET_SIG(pQInfo); + sem_post(&pQInfo->dataReady); + return; + } + } + } + pQInfo->over = 1; dTrace("QInfo:%p vid:%d sid:%d id:%s, query over, %d points are returned, reset signature", pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQInfo->pointsRead); @@ -1091,21 +1238,20 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { int64_t st = taosGetTimestampUs(); - if (pQuery->nAggTimeInterval != 0) { // interval (downsampling operation) - assert(pQuery->nAggTimeInterval != 0 && pQuery->checkBufferInLoop == 0 && - pQuery->pointsOffset == pQuery->pointsToRead); + if (pQuery->nAggTimeInterval != 0) { // interval (down sampling operation) + assert(pQuery->checkBufferInLoop == 0 && pQuery->pointsOffset == pQuery->pointsToRead); vnodeSingleMeterIntervalProcessor(pQInfo); } else { if (isFixedOutputQuery(pQuery)) { - assert(pQuery->checkBufferInLoop == 0 && pQuery->nAggTimeInterval == 0); + assert(pQuery->checkBufferInLoop == 0); vnodeSingleMeterFixedOutputProcessor(pQInfo); } else { // diff/add/multiply/subtract/division - assert(pQuery->checkBufferInLoop == 1 && pQuery->nAggTimeInterval == 0); + assert(pQuery->checkBufferInLoop == 1); vnodeSingleMeterMultiOutputProcessor(pQInfo); } } - /* record the total elapsed time */ + // record the total elapsed time pQInfo->useconds += (taosGetTimestampUs() - st); /* check if query is killed or not */ @@ -1139,27 +1285,15 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) { SQuery *pQuery = &pQInfo->query; pQuery->pointsRead = 0; - SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; - if (pSupporter->meterIdx >= pSupporter->pSidSet->numOfSids) { - pQInfo->over = 1; - dTrace("QInfo:%p over, %d meters queried, %d points are returned, reset signature", pQInfo, pSupporter->numOfMeters, - pQInfo->pointsRead); - - // reset status - TSDB_QINFO_RESET_SIG(pQInfo); - - vnodePrintQueryStatistics(pSupporter); - sem_post(&pQInfo->dataReady); - return; - } - int64_t st = taosGetTimestampUs(); - if (pQuery->nAggTimeInterval > 0 || (isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)))) { + if (pQuery->nAggTimeInterval > 0 || + (isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)) && !isGroupbyNormalCol(pQuery->pGroupbyExpr))) { assert(pQuery->checkBufferInLoop == 0); vnodeMultiMeterQueryProcessor(pQInfo); } else { - assert((pQuery->checkBufferInLoop == 1 && pQuery->nAggTimeInterval == 0 && pQuery->pGroupbyExpr == NULL) || - isPointInterpoQuery(pQuery)); + assert((pQuery->checkBufferInLoop == 1 && pQuery->nAggTimeInterval == 0) || isPointInterpoQuery(pQuery) || + isGroupbyNormalCol(pQuery->pGroupbyExpr)); + vnodeMultiMeterMultiOutputProcessor(pQInfo); } @@ -1171,6 +1305,15 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) { taosInterpoSetStartInfo(&pQInfo->pMeterQuerySupporter->runtimeEnv.interpoInfo, pQuery->pointsRead, pQInfo->query.interpoType); + SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; + + if (pQuery->pointsRead == 0) { + pQInfo->over = 1; + dTrace("QInfo:%p over, %d meters queried, %d points are returned, reset signature", pQInfo, pSupporter->numOfMeters, + pQInfo->pointsRead); + vnodePrintQueryStatistics(pSupporter); + } + TSDB_QINFO_RESET_SIG(pQInfo); sem_post(&pQInfo->dataReady); } diff --git a/src/system/src/vnodeRead.c b/src/system/detail/src/vnodeRead.c similarity index 83% rename from src/system/src/vnodeRead.c rename to src/system/detail/src/vnodeRead.c index 98bac17f67..3e580b747f 100644 --- a/src/system/src/vnodeRead.c +++ b/src/system/detail/src/vnodeRead.c @@ -13,8 +13,11 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include +#include +#include #include #include @@ -22,9 +25,12 @@ #include "taosmsg.h" #include "tast.h" #include "textbuffer.h" +#include "tscJoinProcess.h" +#include "tscompression.h" #include "vnode.h" #include "vnodeRead.h" #include "vnodeUtil.h" + #pragma GCC diagnostic ignored "-Wint-conversion" int (*pQueryFunc[])(SMeterObj *, SQuery *) = {vnodeQueryFromCache, vnodeQueryFromFile}; @@ -157,7 +163,7 @@ static SQInfo *vnodeAllocateQInfoCommon(SQueryMeterMsg *pQueryMsg, SMeterObj *pM SQuery *pQuery = &(pQInfo->query); - SColumnFilterMsg *colList = pQueryMsg->colList; + SColumnInfo *colList = pQueryMsg->colList; short numOfCols = pQueryMsg->numOfCols; short numOfOutputCols = pQueryMsg->numOfOutputCols; @@ -171,7 +177,7 @@ static SQInfo *vnodeAllocateQInfoCommon(SQueryMeterMsg *pQueryMsg, SMeterObj *pM pQuery->order.order = pQueryMsg->order; pQuery->order.orderColId = pQueryMsg->orderColId; - pQuery->colList = calloc(1, sizeof(SColumnFilter) * numOfCols); + pQuery->colList = calloc(1, sizeof(SSingleColumnFilterInfo) * numOfCols); if (pQuery->colList == NULL) { goto _clean_memory; } @@ -179,7 +185,22 @@ static SQInfo *vnodeAllocateQInfoCommon(SQueryMeterMsg *pQueryMsg, SMeterObj *pM for (int16_t i = 0; i < numOfCols; ++i) { pQuery->colList[i].req[0] = 1; // column required during mater scan of data blocks pQuery->colList[i].colIdxInBuf = i; + pQuery->colList[i].data = colList[i]; + SColumnInfo *pColInfo = &pQuery->colList[i].data; + + pColInfo->filters = NULL; + + if (colList[i].numOfFilters > 0) { + pColInfo->filters = calloc(1, colList[i].numOfFilters * sizeof(SColumnFilterInfo)); + + for (int32_t j = 0; j < colList[i].numOfFilters; ++j) { + tscColumnFilterInfoCopy(&pColInfo->filters[j], &colList[i].filters[j]); + } + } else { + pQuery->colList[i].data.filters = NULL; + } + pQuery->dataRowSize += colList[i].bytes; } @@ -189,15 +210,14 @@ static SQInfo *vnodeAllocateQInfoCommon(SQueryMeterMsg *pQueryMsg, SMeterObj *pM assert(pExprs[col].resBytes > 0); pQuery->rowSize += pExprs[col].resBytes; - if (pExprs[col].pBase.colInfo.isTag) { + if (TSDB_COL_IS_TAG(pExprs[col].pBase.colInfo.flag)) { continue; } int16_t colId = pExprs[col].pBase.colInfo.colId; int16_t functId = pExprs[col].pBase.functionId; - // build the projection of actual column data in buffer and the real column - // index + // build the projection of actual column data in buffer and the real column index for (int32_t k = 0; k < numOfCols; ++k) { if (pQuery->colList[k].data.colId == colId) { pExprs[col].pBase.colInfo.colIdxInBuf = (int16_t)k; @@ -260,7 +280,10 @@ static SQInfo *vnodeAllocateQInfoEx(SQueryMeterMsg *pQueryMsg, SSqlGroupbyExpr * pQInfo->query.pointsToRead = vnodeList[pMeterObj->vnode].cfg.rowsInFileBlock; for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { - size_t size = (pQInfo->query.pointsToRead + 1) * pExprs[col].resBytes + sizeof(SData); + assert(pExprs[col].interResBytes >= pExprs[col].resBytes); + + // allocate additional memory for interResults that are usually larger then final results + size_t size = (pQInfo->query.pointsToRead + 1) * pExprs[col].resBytes + pExprs[col].interResBytes + sizeof(SData); pQuery->sdata[col] = (SData *)calloc(1, size); if (pQuery->sdata[col] == NULL) { goto sign_clean_memory; @@ -321,7 +344,7 @@ SQInfo *vnodeAllocateQInfo(SQueryMeterMsg *pQueryMsg, SMeterObj *pObj, SSqlFunct SQuery *pQuery = &(pQInfo->query); - pQuery->sdata = (SData **)malloc(sizeof(SData *) * pQuery->numOfOutputCols); + pQuery->sdata = (SData **)calloc(1, sizeof(SData *) * pQuery->numOfOutputCols); if (pQuery->sdata == NULL) { goto __clean_memory; } @@ -426,10 +449,7 @@ void vnodeFreeQInfo(void *param, bool decQueryRef) { } for (int col = 0; col < pQuery->numOfCols; ++col) { - if (pQuery->colList[col].data.filterOnBinary == 1 && pQuery->colList[col].data.filterOn) { - tfree(pQuery->colList[col].data.pz); - pQuery->colList[col].data.len = 0; - } + vnodeFreeColumnInfo(&pQuery->colList[col].data); } if (pQuery->colList[0].colIdx != PRIMARYKEY_TIMESTAMP_COL_INDEX) { @@ -439,6 +459,13 @@ void vnodeFreeQInfo(void *param, bool decQueryRef) { sem_destroy(&(pQInfo->dataReady)); vnodeQueryFreeQInfoEx(pQInfo); + for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { + SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; + if (pColFilter->numOfFilters > 0) { + tfree(pColFilter->pFilters); + } + } + tfree(pQuery->pFilterInfo); tfree(pQuery->colList); tfree(pQuery->sdata); @@ -449,7 +476,7 @@ void vnodeFreeQInfo(void *param, bool decQueryRef) { if (pBinExprInfo->numOfCols > 0) { tfree(pBinExprInfo->pReqColumns); - tSQLBinaryExprDestroy(&pBinExprInfo->pBinExpr); + tSQLBinaryExprDestroy(&pBinExprInfo->pBinExpr, NULL); } } @@ -503,8 +530,8 @@ void vnodeQueryData(SSchedMsg *pMsg) { SMeterObj *pObj = pQInfo->pObj; - dTrace("QInfo:%p vid:%d sid:%d id:%s, query thread is created, numOfQueries:%d", pQInfo, pObj->vnode, pObj->sid, - pObj->meterId, pObj->numOfQueries); + dTrace("QInfo:%p vid:%d sid:%d id:%s, query thread is created, numOfQueries:%d, func:%s", pQInfo, pObj->vnode, + pObj->sid, pObj->meterId, pObj->numOfQueries, __FUNCTION__); pQuery->pointsToRead = vnodeList[pObj->vnode].cfg.rowsInFileBlock; pQuery->pointsOffset = pQInfo->bufIndex * pQuery->pointsToRead; @@ -544,15 +571,15 @@ void vnodeQueryData(SSchedMsg *pMsg) { pQuery->slot = -1; // reset the handle pQuery->over = 0; - dTrace("vid:%d sid:%d id:%s, query in other media, order:%d, skey:%lld query:%p", - pObj->vnode, pObj->sid, pObj->meterId, pQuery->order.order, pQuery->skey, pQuery); + dTrace("vid:%d sid:%d id:%s, query in other media, order:%d, skey:%lld query:%p", pObj->vnode, pObj->sid, + pObj->meterId, pQuery->order.order, pQuery->skey, pQuery); } pQInfo->pointsRead += pQuery->pointsRead; - dTrace("vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d last key:%lld, query:%p", - pObj->vnode, pObj->sid, pObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned, - pQuery->lastKey, pQuery); + dTrace("vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d last key:%lld, query:%p", pObj->vnode, + pObj->sid, pObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned, pQuery->lastKey, + pQuery); int64_t et = taosGetTimestampUs(); pQInfo->useconds += et - st; @@ -580,6 +607,7 @@ void *vnodeQueryInTimeRange(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyExp SMeterObj *pMeterObj = pMetersObj[0]; bool isProjQuery = vnodeIsProjectionQuery(pSqlExprs, pQueryMsg->numOfOutputCols); + // todo pass the correct error code if (isProjQuery) { pQInfo = vnodeAllocateQInfo(pQueryMsg, pMeterObj, pSqlExprs); } else { @@ -628,7 +656,16 @@ void *vnodeQueryInTimeRange(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyExp pQInfo->pMeterQuerySupporter = pSupporter; - if (((*code) = vnodeQuerySingleMeterPrepare(pQInfo, pQInfo->pObj, pSupporter)) != TSDB_CODE_SUCCESS) { + STSBuf *pTSBuf = NULL; + if (pQueryMsg->tsLen > 0) { + // open new file to save the result + char *tsBlock = (char *)pQueryMsg + pQueryMsg->tsOffset; + pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder); + tsBufResetPos(pTSBuf); + tsBufNextPos(pTSBuf); + } + + if (((*code) = vnodeQuerySingleMeterPrepare(pQInfo, pQInfo->pObj, pSupporter, pTSBuf)) != TSDB_CODE_SUCCESS) { goto _error; } @@ -667,7 +704,7 @@ void *vnodeQueryOnMultiMeters(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE SQInfo *pQInfo; SQuery *pQuery; - assert(pQueryMsg->metricQuery == 1 && pQueryMsg->numOfCols > 0 && pQueryMsg->pSidExtInfo != 0 && + assert(QUERY_IS_STABLE_QUERY(pQueryMsg->queryType) && pQueryMsg->numOfCols > 0 && pQueryMsg->pSidExtInfo != 0 && pQueryMsg->numOfSids >= 1); pQInfo = vnodeAllocateQInfoEx(pQueryMsg, pGroupbyExpr, pSqlExprs, *pMetersObj); @@ -726,10 +763,10 @@ void *vnodeQueryOnMultiMeters(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE px += sidElemLen; } - if (pGroupbyExpr != NULL && pGroupbyExpr->numOfGroupbyCols > 0) { + if (pGroupbyExpr != NULL && pGroupbyExpr->numOfGroupCols > 0) { pSupporter->pSidSet = tSidSetCreate(pSupporter->pMeterSidExtInfo, pQueryMsg->numOfSids, (SSchema *)pQueryMsg->pTagSchema, - pQueryMsg->numOfTagsCols, pGroupbyExpr->tagIndex, pGroupbyExpr->numOfGroupbyCols); + pQueryMsg->numOfTagsCols, pGroupbyExpr->columnInfo, pGroupbyExpr->numOfGroupCols); } else { pSupporter->pSidSet = tSidSetCreate(pSupporter->pMeterSidExtInfo, pQueryMsg->numOfSids, (SSchema *)pQueryMsg->pTagSchema, pQueryMsg->numOfTagsCols, NULL, 0); @@ -737,7 +774,15 @@ void *vnodeQueryOnMultiMeters(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE pQInfo->pMeterQuerySupporter = pSupporter; - if (((*code) = vnodeMultiMeterQueryPrepare(pQInfo, pQuery)) != TSDB_CODE_SUCCESS) { + STSBuf *pTSBuf = NULL; + if (pQueryMsg->tsLen > 0) { + // open new file to save the result + char *tsBlock = (char *)pQueryMsg + pQueryMsg->tsOffset; + pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder); + tsBufResetPos(pTSBuf); + } + + if (((*code) = vnodeMultiMeterQueryPrepare(pQInfo, pQuery, pTSBuf)) != TSDB_CODE_SUCCESS) { goto _error; } @@ -776,18 +821,24 @@ int vnodeRetrieveQueryInfo(void *handle, int *numOfRows, int *rowSize, int16_t * *rowSize = 0; pQInfo = (SQInfo *)handle; - if (pQInfo == NULL || pQInfo->killed) return -1; - pQuery = &(pQInfo->query); + if (pQInfo == NULL) { + return TSDB_CODE_INVALID_QHANDLE; + } + pQuery = &(pQInfo->query); if (!vnodeIsQInfoValid(pQInfo) || (pQuery->sdata == NULL)) { - dError("%p retrieve memory is corrupted!!! QInfo:%p, sign:%p, sdata:%p", pQuery, pQInfo, pQInfo->signature, - pQuery->sdata); - return TSDB_CODE_APP_ERROR; + dError("QInfo:%p %p retrieve memory is corrupted!!! QInfo:%p, sign:%p, sdata:%p", pQInfo, pQuery, pQInfo, + pQInfo->signature, pQuery->sdata); + return TSDB_CODE_INVALID_QHANDLE; } if (pQInfo->killed) { - dTrace("%p it is already killed", pQuery); - return TSDB_CODE_APP_ERROR; + dTrace("QInfo:%p it is already killed, %p, code:%d", pQInfo, pQuery, pQInfo->code); + if (pQInfo->code == TSDB_CODE_SUCCESS) { + return TSDB_CODE_QUERY_CANCELLED; + } else { // in case of not TSDB_CODE_SUCCESS, return the code to client + return pQInfo->code; + } } sem_wait(&pQInfo->dataReady); @@ -798,18 +849,18 @@ int vnodeRetrieveQueryInfo(void *handle, int *numOfRows, int *rowSize, int16_t * if (pQInfo->code < 0) return -pQInfo->code; - return 0; + return TSDB_CODE_SUCCESS; } // vnodeRetrieveQueryInfo must be called first -int vnodeSaveQueryResult(void *handle, char *data) { +int vnodeSaveQueryResult(void *handle, char *data, int32_t *size) { SQInfo *pQInfo = (SQInfo *)handle; // the remained number of retrieved rows, not the interpolated result int numOfRows = pQInfo->pointsRead - pQInfo->pointsReturned; - int32_t numOfFinal = vnodeCopyQueryResultToMsg(pQInfo, data, numOfRows); - pQInfo->pointsReturned += numOfFinal; //(pQInfo->pointsRead + pQInfo->pointsInterpo); + int32_t numOfFinal = vnodeCopyQueryResultToMsg(pQInfo, data, numOfRows, size); + pQInfo->pointsReturned += numOfFinal; dTrace("QInfo:%p %d are returned, totalReturned:%d totalRead:%d", pQInfo, numOfFinal, pQInfo->pointsReturned, pQInfo->pointsRead); @@ -860,7 +911,7 @@ static int32_t validateQueryMeterMsg(SQueryMeterMsg *pQueryMsg) { return -1; } - if (pQueryMsg->numOfTagsCols < 0 || pQueryMsg->numOfTagsCols > TSDB_MAX_TAGS) { + if (pQueryMsg->numOfTagsCols < 0 || pQueryMsg->numOfTagsCols > TSDB_MAX_TAGS + 1) { dError("qmsg:%p illegal value of numOfTagsCols %ld", pQueryMsg, pQueryMsg->numOfTagsCols); return -1; } @@ -875,8 +926,8 @@ static int32_t validateQueryMeterMsg(SQueryMeterMsg *pQueryMsg) { return -1; } - if (pQueryMsg->numOfGroupbyCols < 0) { - dError("qmsg:%p illegal value of numOfGroupbyCols %ld", pQueryMsg, pQueryMsg->numOfGroupbyCols); + if (pQueryMsg->numOfGroupCols < 0) { + dError("qmsg:%p illegal value of numOfGroupbyCols %ld", pQueryMsg, pQueryMsg->numOfGroupCols); return -1; } @@ -910,17 +961,21 @@ int32_t vnodeConvertQueryMeterMsg(SQueryMeterMsg *pQueryMsg) { pQueryMsg->order = htons(pQueryMsg->order); pQueryMsg->orderColId = htons(pQueryMsg->orderColId); - pQueryMsg->metricQuery = htons(pQueryMsg->metricQuery); + pQueryMsg->queryType = htons(pQueryMsg->queryType); pQueryMsg->nAggTimeInterval = htobe64(pQueryMsg->nAggTimeInterval); pQueryMsg->numOfTagsCols = htons(pQueryMsg->numOfTagsCols); pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); pQueryMsg->numOfOutputCols = htons(pQueryMsg->numOfOutputCols); - pQueryMsg->numOfGroupbyCols = htons(pQueryMsg->numOfGroupbyCols); + pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); pQueryMsg->tagLength = htons(pQueryMsg->tagLength); pQueryMsg->limit = htobe64(pQueryMsg->limit); pQueryMsg->offset = htobe64(pQueryMsg->offset); + pQueryMsg->tsOffset = htonl(pQueryMsg->tsOffset); + pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); + pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); + pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); // query msg safety check if (validateQueryMeterMsg(pQueryMsg) != 0) { @@ -928,30 +983,43 @@ int32_t vnodeConvertQueryMeterMsg(SQueryMeterMsg *pQueryMsg) { } SMeterSidExtInfo **pSids = NULL; - char * strBuf = (char *)&pQueryMsg->colList[pQueryMsg->numOfCols]; + char * pMsg = (char *)(pQueryMsg->colList) + sizeof(SColumnInfo) * pQueryMsg->numOfCols; + for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) { pQueryMsg->colList[col].colId = htons(pQueryMsg->colList[col].colId); - pQueryMsg->colList[col].type = htons(pQueryMsg->colList[col].type); pQueryMsg->colList[col].bytes = htons(pQueryMsg->colList[col].bytes); + pQueryMsg->colList[col].numOfFilters = htons(pQueryMsg->colList[col].numOfFilters); assert(pQueryMsg->colList[col].type >= TSDB_DATA_TYPE_BOOL && pQueryMsg->colList[col].type <= TSDB_DATA_TYPE_NCHAR); - pQueryMsg->colList[col].filterOn = htons(pQueryMsg->colList[col].filterOn); - pQueryMsg->colList[col].filterOnBinary = htons(pQueryMsg->colList[col].filterOnBinary); + int32_t numOfFilters = pQueryMsg->colList[col].numOfFilters; - if (pQueryMsg->colList[col].filterOn && pQueryMsg->colList[col].filterOnBinary) { - pQueryMsg->colList[col].len = htobe64(pQueryMsg->colList[col].len); - pQueryMsg->colList[col].pz = calloc(1, pQueryMsg->colList[col].len + 1); - strcpy(pQueryMsg->colList[col].pz, strBuf); - strBuf += pQueryMsg->colList[col].len + 1; - } else { - pQueryMsg->colList[col].lowerBndi = htobe64(pQueryMsg->colList[col].lowerBndi); - pQueryMsg->colList[col].upperBndi = htobe64(pQueryMsg->colList[col].upperBndi); + if (numOfFilters > 0) { + pQueryMsg->colList[col].filters = calloc(numOfFilters, sizeof(SColumnFilterInfo)); } - pQueryMsg->colList[col].lowerRelOptr = htons(pQueryMsg->colList[col].lowerRelOptr); - pQueryMsg->colList[col].upperRelOptr = htons(pQueryMsg->colList[col].upperRelOptr); + for (int32_t f = 0; f < numOfFilters; ++f) { + SColumnFilterInfo *pFilterInfo = (SColumnFilterInfo *)pMsg; + SColumnFilterInfo *pDestFilterInfo = &pQueryMsg->colList[col].filters[f]; + + pDestFilterInfo->filterOnBinary = htons(pFilterInfo->filterOnBinary); + + pMsg += sizeof(SColumnFilterInfo); + + if (pDestFilterInfo->filterOnBinary) { + pDestFilterInfo->len = htobe64(pFilterInfo->len); + pDestFilterInfo->pz = calloc(1, pDestFilterInfo->len + 1); + memcpy(pDestFilterInfo->pz, pMsg, pDestFilterInfo->len + 1); + pMsg += (pDestFilterInfo->len + 1); + } else { + pDestFilterInfo->lowerBndi = htobe64(pFilterInfo->lowerBndi); + pDestFilterInfo->upperBndi = htobe64(pFilterInfo->upperBndi); + } + + pDestFilterInfo->lowerRelOptr = htons(pFilterInfo->lowerRelOptr); + pDestFilterInfo->upperRelOptr = htons(pFilterInfo->upperRelOptr); + } } bool hasArithmeticFunction = false; @@ -962,7 +1030,6 @@ int32_t vnodeConvertQueryMeterMsg(SQueryMeterMsg *pQueryMsg) { */ pQueryMsg->pSqlFuncExprs = malloc(POINTER_BYTES * pQueryMsg->numOfOutputCols); - char * pMsg = strBuf; SSqlFuncExprMsg *pExprMsg = (SSqlFuncExprMsg *)pMsg; for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { @@ -970,6 +1037,7 @@ int32_t vnodeConvertQueryMeterMsg(SQueryMeterMsg *pQueryMsg) { pExprMsg->colInfo.colIdx = htons(pExprMsg->colInfo.colIdx); pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); + pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); pExprMsg->functionId = htons(pExprMsg->functionId); pExprMsg->numOfParams = htons(pExprMsg->numOfParams); @@ -990,9 +1058,10 @@ int32_t vnodeConvertQueryMeterMsg(SQueryMeterMsg *pQueryMsg) { if (pExprMsg->functionId == TSDB_FUNC_ARITHM) { hasArithmeticFunction = true; - } else if (pExprMsg->functionId == TSDB_FUNC_TAG || pExprMsg->functionId == TSDB_FUNC_TAGPRJ) { - // ignore the column index check for arithmetic expression. - if (!pExprMsg->colInfo.isTag) { + } else if (pExprMsg->functionId == TSDB_FUNC_TAG || + pExprMsg->functionId == TSDB_FUNC_TAGPRJ || + pExprMsg->functionId == TSDB_FUNC_TAG_DUMMY) { + if (pExprMsg->colInfo.flag != TSDB_COL_TAG) { // ignore the column index check for arithmetic expression. return TSDB_CODE_INVALID_QUERY_MSG; } } else { @@ -1025,12 +1094,12 @@ int32_t vnodeConvertQueryMeterMsg(SQueryMeterMsg *pQueryMsg) { pMsg = (char *)pSids[pQueryMsg->numOfSids - 1]; pMsg += sizeof(SMeterSidExtInfo) + pQueryMsg->tagLength; - if (pQueryMsg->numOfGroupbyCols > 0 || pQueryMsg->numOfTagsCols > 0) { // group by tag columns + if (pQueryMsg->numOfGroupCols > 0 || pQueryMsg->numOfTagsCols > 0) { // group by tag columns pQueryMsg->pTagSchema = (uint64_t)pMsg; SSchema *pTagSchema = (SSchema *)pQueryMsg->pTagSchema; pMsg += sizeof(SSchema) * pQueryMsg->numOfTagsCols; - if (pQueryMsg->numOfGroupbyCols > 0) { + if (pQueryMsg->numOfGroupCols > 0) { pQueryMsg->groupbyTagIds = (uint64_t) & (pTagSchema[pQueryMsg->numOfTagsCols]); } else { pQueryMsg->groupbyTagIds = 0; @@ -1038,7 +1107,7 @@ int32_t vnodeConvertQueryMeterMsg(SQueryMeterMsg *pQueryMsg) { pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); pQueryMsg->orderType = htons(pQueryMsg->orderType); - pMsg += sizeof(int16_t) * pQueryMsg->numOfGroupbyCols; + pMsg += sizeof(SColIndexEx) * pQueryMsg->numOfGroupCols; } else { pQueryMsg->pTagSchema = 0; pQueryMsg->groupbyTagIds = 0; @@ -1055,10 +1124,12 @@ int32_t vnodeConvertQueryMeterMsg(SQueryMeterMsg *pQueryMsg) { } dTrace("qmsg:%p query on %d meter(s), qrange:%lld-%lld, numOfGroupbyTagCols:%d, numOfTagCols:%d, timestamp order:%d, " - "tags order:%d, tags order col:%d, numOfOutputCols:%d, numOfCols:%d, interval:%lld, fillType:%d", - pQueryMsg, pQueryMsg->numOfSids, pQueryMsg->skey, pQueryMsg->ekey, pQueryMsg->numOfGroupbyCols, + "tags order:%d, tags order col:%d, numOfOutputCols:%d, numOfCols:%d, interval:%lld, fillType:%d, comptslen:%d, limit:%lld, " + "offset:%lld", + pQueryMsg, pQueryMsg->numOfSids, pQueryMsg->skey, pQueryMsg->ekey, pQueryMsg->numOfGroupCols, pQueryMsg->numOfTagsCols, pQueryMsg->order, pQueryMsg->orderType, pQueryMsg->orderByIdx, - pQueryMsg->numOfOutputCols, pQueryMsg->numOfCols, pQueryMsg->nAggTimeInterval, pQueryMsg->interpoType); + pQueryMsg->numOfOutputCols, pQueryMsg->numOfCols, pQueryMsg->nAggTimeInterval, pQueryMsg->interpoType, + pQueryMsg->tsLen, pQueryMsg->limit, pQueryMsg->offset); return 0; } diff --git a/src/system/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c similarity index 86% rename from src/system/src/vnodeShell.c rename to src/system/detail/src/vnodeShell.c index 66818766d9..190c1d6ee6 100644 --- a/src/system/src/vnodeShell.c +++ b/src/system/detail/src/vnodeShell.c @@ -14,7 +14,6 @@ */ #define _DEFAULT_SOURCE - #include #include #include @@ -26,6 +25,7 @@ #include "textbuffer.h" #include "trpc.h" +#include "tscJoinProcess.h" #include "vnode.h" #include "vnodeRead.h" #include "vnodeUtil.h" @@ -88,19 +88,29 @@ void *vnodeProcessMsgFromShell(char *msg, void *ahandle, void *thandle) { } } - dTrace("vid:%d sid:%d, msg:%s is received pConn:%p", vnode, sid, taosMsg[pMsg->msgType], thandle); + // if ( vnodeList[vnode].status != TSDB_STATUS_MASTER && pMsg->msgType != TSDB_MSG_TYPE_RETRIEVE ) { - // set in query processing flag - if (pMsg->msgType == TSDB_MSG_TYPE_QUERY) { - vnodeProcessQueryRequest((char *)pMsg->content, pMsg->msgLen - sizeof(SIntMsg), pObj); - } else if (pMsg->msgType == TSDB_MSG_TYPE_RETRIEVE) { - vnodeProcessRetrieveRequest((char *)pMsg->content, pMsg->msgLen - sizeof(SIntMsg), pObj); - } else if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) { - vnodeProcessShellSubmitRequest((char *)pMsg->content, pMsg->msgLen - sizeof(SIntMsg), pObj); +#ifdef CLUSTER + if (vnodeList[vnode].status != TSDB_STATUS_MASTER) { + taosSendSimpleRsp(thandle, pMsg->msgType + 1, TSDB_CODE_NOT_READY); + dTrace("vid:%d sid:%d, shell msg is ignored since in state:%d", vnode, sid, vnodeList[vnode].status); } else { - dError("%s is not processed", taosMsg[pMsg->msgType]); +#endif + dTrace("vid:%d sid:%d, msg:%s is received pConn:%p", vnode, sid, taosMsg[pMsg->msgType], thandle); + + if (pMsg->msgType == TSDB_MSG_TYPE_QUERY) { + vnodeProcessQueryRequest((char *)pMsg->content, pMsg->msgLen - sizeof(SIntMsg), pObj); + } else if (pMsg->msgType == TSDB_MSG_TYPE_RETRIEVE) { + vnodeProcessRetrieveRequest((char *)pMsg->content, pMsg->msgLen - sizeof(SIntMsg), pObj); + } else if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) { + vnodeProcessShellSubmitRequest((char *)pMsg->content, pMsg->msgLen - sizeof(SIntMsg), pObj); + } else { + dError("%s is not processed", taosMsg[pMsg->msgType]); + } +#ifdef CLUSTER } - +#endif + return pObj; } @@ -118,7 +128,11 @@ int vnodeInitShell() { if (numOfThreads < 1) numOfThreads = 1; memset(&rpcInit, 0, sizeof(rpcInit)); +#ifdef CLUSTER + rpcInit.localIp = tsInternalIp; +#else rpcInit.localIp = "0.0.0.0"; +#endif rpcInit.localPort = tsVnodeShellPort; rpcInit.label = "DND-shell"; rpcInit.numOfThreads = numOfThreads; @@ -142,23 +156,23 @@ int vnodeInitShell() { } int vnodeOpenShellVnode(int vnode) { - SVnodeCfg *pCfg = &vnodeList[vnode].cfg; - int sessions = pCfg->maxSessions * 1.1; - if (sessions < 300) sessions = 300; + const int32_t MIN_NUM_OF_SESSIONS = 300; - int size = sessions * sizeof(SShellObj); + SVnodeCfg *pCfg = &vnodeList[vnode].cfg; + int32_t sessions = (int32_t) MAX(pCfg->maxSessions * 1.1, MIN_NUM_OF_SESSIONS); - shellList[vnode] = (SShellObj *)malloc(size); + size_t size = sessions * sizeof(SShellObj); + shellList[vnode] = (SShellObj *)calloc(1, size); if (shellList[vnode] == NULL) { - dError("vid:%d failed to allocate shellObj", vnode); + dError("vid:%d failed to allocate shellObj, size:%d", vnode, size); return -1; } - memset(shellList[vnode], 0, size); - - taosOpenRpcChannWithQ(pShellServer, vnode, sessions, rpcQhandle[(vnode+1)%tsMaxQueues]); + if(taosOpenRpcChannWithQ(pShellServer, vnode, sessions, rpcQhandle[(vnode+1)%tsMaxQueues]) != TSDB_CODE_SUCCESS) { + return -1; + } - return 0; + return TSDB_CODE_SUCCESS; } static void vnodeDelayedFreeResource(void *param, void *tmrId) { @@ -184,7 +198,7 @@ void vnodeCloseShellVnode(int vnode) { * 1. The msg, as well as SRpcConn may be in the task queue, free it immediate will cause crash * 2. Free connection may cause *(SRpcConn*)pObj->thandle to be invalid to access. */ - dTrace("vid:%d, delay 500ms to free resources", vnode); + dTrace("vid:%d, free resources in 500ms", vnode); taosTmrStart(vnodeDelayedFreeResource, 500, v, vnodeTmrCtrl); } @@ -320,7 +334,7 @@ int vnodeProcessQueryRequest(char *pMsg, int msgLen, SShellObj *pObj) { } pGroupbyExpr = vnodeCreateGroupbyExpr(pQueryMsg, &code); - if ((pGroupbyExpr == NULL && pQueryMsg->numOfGroupbyCols != 0) || code != TSDB_CODE_SUCCESS) { + if ((pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { goto _query_over; } @@ -330,23 +344,26 @@ int vnodeProcessQueryRequest(char *pMsg, int msgLen, SShellObj *pObj) { pObj->qhandle = NULL; } - if (pQueryMsg->metricQuery) { + if (QUERY_IS_STABLE_QUERY(pQueryMsg->queryType)) { pObj->qhandle = vnodeQueryOnMultiMeters(pMeterObjList, pGroupbyExpr, pExprs, pQueryMsg, &code); } else { - assert(pGroupbyExpr == NULL); pObj->qhandle = vnodeQueryInTimeRange(pMeterObjList, pGroupbyExpr, pExprs, pQueryMsg, &code); } _query_over: + // if failed to add ref for all meters in this query, abort current query if (code != TSDB_CODE_SUCCESS) { - // if failed to add ref for all meters in this query, abort current query vnodeDecQueryRefCount(pQueryMsg, pMeterObjList, incNumber); } + tfree(pQueryMsg->pSqlFuncExprs); tfree(pMeterObjList); ret = vnodeSendQueryRspMsg(pObj, code, pObj->qhandle); - free(pSids); + free(pQueryMsg->pSidExtInfo); + for(int32_t i = 0; i < pQueryMsg->numOfCols; ++i) { + vnodeFreeColumnInfo(&pQueryMsg->colList[i]); + } __sync_fetch_and_add(&vnodeSelectReqNum, 1); return ret; @@ -366,6 +383,7 @@ void vnodeExecuteRetrieveReq(SSchedMsg *pSched) { int code = 0; pRetrieve = (SRetrieveMeterMsg *)pMsg; + pRetrieve->free = htons(pRetrieve->free); /* * in case of server restart, apps may hold qhandle created by server before restart, @@ -373,7 +391,7 @@ void vnodeExecuteRetrieveReq(SSchedMsg *pSched) { */ if (pRetrieve->qhandle == (uint64_t)pObj->qhandle) { // if free flag is set, client wants to clean the resources - if (pRetrieve->free == 0) + if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) code = vnodeRetrieveQueryInfo((void *)(pRetrieve->qhandle), &numOfRows, &rowSize, &timePrec); } else { dError("QInfo:%p, qhandle:%p is not matched with saved:%p", pObj->qhandle, pRetrieve->qhandle, pObj->qhandle); @@ -385,7 +403,11 @@ void vnodeExecuteRetrieveReq(SSchedMsg *pSched) { } pStart = taosBuildRspMsgWithSize(pObj->thandle, TSDB_MSG_TYPE_RETRIEVE_RSP, size + 100); - if (pStart == NULL) goto _exit; + if (pStart == NULL) { + taosSendSimpleRsp(pObj->thandle, TSDB_MSG_TYPE_RETRIEVE_RSP, TSDB_CODE_SERV_OUT_OF_MEMORY); + goto _exit; + } + pMsg = pStart; *pMsg = code; @@ -406,7 +428,11 @@ void vnodeExecuteRetrieveReq(SSchedMsg *pSched) { pMsg = pRsp->data; if (numOfRows > 0 && code == TSDB_CODE_SUCCESS) { - vnodeSaveQueryResult((void *)(pRetrieve->qhandle), pRsp->data); + int32_t oldSize = size; + vnodeSaveQueryResult((void *)(pRetrieve->qhandle), pRsp->data, &size); + if (oldSize > size) { + pRsp->compress = htons(1); // denote that the response msg is compressed + } } pMsg += size; @@ -485,6 +511,7 @@ int vnodeProcessShellSubmitRequest(char *pMsg, int msgLen, SShellObj *pObj) { int32_t numOfPoints = 0; int32_t numOfTotalPoints = 0; + // We take current time here to avoid it in the for loop. TSKEY now = taosGetTimestamp(pVnode->cfg.precision); for (int32_t i = 0; i < pSubmit->numOfSid; ++i) { diff --git a/src/system/src/vnodeStore.c b/src/system/detail/src/vnodeStore.c similarity index 85% rename from src/system/src/vnodeStore.c rename to src/system/detail/src/vnodeStore.c index bd8f251c37..e00b4de7bc 100644 --- a/src/system/src/vnodeStore.c +++ b/src/system/detail/src/vnodeStore.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include #include @@ -26,7 +27,8 @@ #include "vnodeStore.h" #include "vnodeUtil.h" -int vnodeCreateMeterObjFile(int vnode); +#pragma GCC diagnostic push +#pragma GCC diagnostic warning "-Woverflow" int tsMaxVnode = -1; int tsOpenVnodes = 0; @@ -49,6 +51,8 @@ int vnodeInitStoreVnode(int vnode) { if (vnodeInitFile(vnode) < 0) return -1; + // vnodeOpenMeterMgmtStoreVnode(vnode); + if (vnodeInitCommit(vnode) < 0) { dError("vid:%d, commit init failed.", pVnode->vnode); return -1; @@ -62,21 +66,36 @@ int vnodeInitStoreVnode(int vnode) { } int vnodeOpenVnode(int vnode) { + int32_t code = TSDB_CODE_SUCCESS; + SVnodeObj *pVnode = vnodeList + vnode; pVnode->vnode = vnode; pVnode->accessState = TSDB_VN_ALL_ACCCESS; + + // vnode is empty if (pVnode->cfg.maxSessions == 0) return 0; pthread_mutex_lock(&dmutex); - vnodeOpenShellVnode(vnode); + // vnodeOpenMeterMgmtVnode(vnode); + + // not enough memory, abort + if ((code = vnodeOpenShellVnode(vnode)) != TSDB_CODE_SUCCESS) { + pthread_mutex_unlock(&dmutex); + return code; + } + + vnodeOpenPeerVnode(vnode); if (vnode > tsMaxVnode) tsMaxVnode = vnode; + vnodeCalcOpenVnodes(); pthread_mutex_unlock(&dmutex); +#ifndef CLUSTER vnodeOpenStreams(pVnode, NULL); +#endif dTrace("vid:%d, vnode is opened, openVnodes:%d", vnode, tsOpenVnodes); @@ -123,6 +142,7 @@ int vnodeCloseVnode(int vnode) { vnodeCloseStream(vnodeList + vnode); vnodeCancelCommit(vnodeList + vnode); + vnodeClosePeerVnode(vnode); vnodeCloseMetersVnode(vnode); vnodeCloseShellVnode(vnode); vnodeCloseCachePool(vnode); @@ -186,6 +206,10 @@ void vnodeRemoveDataFiles(int vnode) { (de->d_type & DT_LNK)) { sprintf(linkFile, "%s/%s", vnodeDir, de->d_name); + if (!vnodeRemoveDataFileFromLinkFile(linkFile, de->d_name)) { + continue; + } + memset(dfilePath, 0, TSDB_FILENAME_LEN); int tcode = readlink(linkFile, dfilePath, TSDB_FILENAME_LEN); remove(linkFile); @@ -238,6 +262,8 @@ int vnodeInitStore() { if (vnodeList == NULL) return -1; memset(vnodeList, 0, size); + if (vnodeInitInfo() < 0) return -1; + for (vnode = 0; vnode < TSDB_MAX_VNODES; ++vnode) { if (vnodeInitStoreVnode(vnode) < 0) { // one vnode is failed to recover from commit log, continue for remain @@ -258,6 +284,34 @@ int vnodeInitVnodes() { return 0; } +void vnodeCleanUpOneVnode(int vnode) { + static int again = 0; + if (vnodeList == NULL) return; + + pthread_mutex_lock(&dmutex); + + if (again) { + pthread_mutex_unlock(&dmutex); + return; + } + again = 1; + + if (vnodeList[vnode].pCachePool) { + vnodeList[vnode].status = TSDB_STATUS_OFFLINE; + vnodeClosePeerVnode(vnode); + } + + pthread_mutex_unlock(&dmutex); + + if (vnodeList[vnode].pCachePool) { + vnodeProcessCommitTimer(vnodeList + vnode, NULL); + while (vnodeList[vnode].commitThread != 0) { + taosMsleep(10); + } + vnodeCleanUpCommit(vnode); + } +} + void vnodeCleanUpVnodes() { static int again = 0; if (vnodeList == NULL) return; @@ -273,6 +327,7 @@ void vnodeCleanUpVnodes() { for (int vnode = 0; vnode < TSDB_MAX_VNODES; ++vnode) { if (vnodeList[vnode].pCachePool) { vnodeList[vnode].status = TSDB_STATUS_OFFLINE; + vnodeClosePeerVnode(vnode); } } @@ -298,3 +353,10 @@ void vnodeCalcOpenVnodes() { __sync_val_compare_and_swap(&tsOpenVnodes, tsOpenVnodes, openVnodes); } + +void vnodeUpdateHeadFile(int vnode, int oldTables, int newTables) { + //todo rewrite the head file with newTables +} + +#pragma GCC diagnostic pop + diff --git a/src/system/src/vnodeStream.c b/src/system/detail/src/vnodeStream.c similarity index 88% rename from src/system/src/vnodeStream.c rename to src/system/detail/src/vnodeStream.c index 2f8cfec1f1..0667ee77bd 100644 --- a/src/system/src/vnodeStream.c +++ b/src/system/detail/src/vnodeStream.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "taosmsg.h" #include "vnode.h" #include "vnodeUtil.h" @@ -79,6 +80,7 @@ void vnodeOpenStreams(void *param, void *tmrId) { SVnodeObj *pVnode = (SVnodeObj *)param; SMeterObj *pObj; + if (pVnode->streamRole == 0) return; if (pVnode->meterList == NULL) return; taosTmrStopA(&pVnode->streamTimer); @@ -99,7 +101,7 @@ void vnodeOpenStreams(void *param, void *tmrId) { } if (pVnode->dbConn == NULL) { - dError("vid:%d, failed to connect to mgmt node: %s", pVnode->vnode, tsInternalIp); + dError("vid:%d, failed to connect to mgmt node", pVnode->vnode); taosTmrReset(vnodeOpenStreams, 1000, param, vnodeTmrCtrl, &pVnode->streamTimer); return; } @@ -117,6 +119,7 @@ void vnodeCreateStream(SMeterObj *pObj) { SVnodeObj *pVnode = vnodeList + pObj->vnode; + if (pVnode->streamRole == 0) return; if (pObj->pStream) return; dTrace("vid:%d sid:%d id:%s stream:%s is created", pObj->vnode, pObj->sid, pObj->meterId, pObj->pSql); @@ -151,7 +154,7 @@ void vnodeRemoveStream(SMeterObj *pObj) { // Close all streams in a vnode void vnodeCloseStream(SVnodeObj *pVnode) { SMeterObj *pObj; - dTrace("vid:%d, stream is closed", pVnode->vnode); + dTrace("vid:%d, stream is closed, old role:%d", pVnode->vnode, pVnode->streamRole); // stop stream computing for (int sid = 0; sid < pVnode->cfg.maxSessions; ++sid) { @@ -165,6 +168,23 @@ void vnodeCloseStream(SVnodeObj *pVnode) { } } +void vnodeUpdateStreamRole(SVnodeObj *pVnode) { + /* SMeterObj *pObj; */ + + int newRole = (pVnode->status == TSDB_STATUS_MASTER) ? 1 : 0; + if (newRole != pVnode->streamRole) { + dTrace("vid:%d, stream role is changed to:%d", pVnode->vnode, newRole); + pVnode->streamRole = newRole; + if (newRole) { + vnodeOpenStreams(pVnode, NULL); + } else { + vnodeCloseStream(pVnode); + } + } else { + dTrace("vid:%d, stream role is keep to:%d", pVnode->vnode, newRole); + } +} + // Callback function called from client void vnodeCloseStreamCallback(void *param) { SMeterObj *pMeter = (SMeterObj *)param; diff --git a/src/system/src/vnodeSystem.c b/src/system/detail/src/vnodeSystem.c similarity index 62% rename from src/system/src/vnodeSystem.c rename to src/system/detail/src/vnodeSystem.c index ef0c9d2d75..2f350db3fa 100644 --- a/src/system/src/vnodeSystem.c +++ b/src/system/detail/src/vnodeSystem.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include #include @@ -26,33 +27,45 @@ #include "tsdb.h" #include "tsocket.h" #include "vnode.h" +#include "vnodeSystem.h" // internal global, not configurable void * vnodeTmrCtrl; void ** rpcQhandle; void * dmQhandle; void * queryQhandle; +int tsVnodePeers = TSDB_VNODES_SUPPORT - 1; int tsMaxQueues; uint32_t tsRebootTime; -int vnodeInitSystem() { - int numOfThreads; +void vnodeCleanUpSystem() { + vnodeCleanUpVnodes(); +} - numOfThreads = tsRatioOfQueryThreads * tsNumOfCores * tsNumOfThreadsPerCore; +bool vnodeInitQueryHandle() { + int numOfThreads = tsRatioOfQueryThreads * tsNumOfCores * tsNumOfThreadsPerCore; if (numOfThreads < 1) numOfThreads = 1; queryQhandle = taosInitScheduler(tsNumOfVnodesPerCore * tsNumOfCores * tsSessionsPerVnode, numOfThreads, "query"); + return true; +} - tsMaxQueues = (1.0 - tsRatioOfQueryThreads) * tsNumOfCores * tsNumOfThreadsPerCore / 2.0; - if (tsMaxQueues < 1) tsMaxQueues = 1; +bool vnodeInitTmrCtl() { + vnodeTmrCtrl = taosTmrInit(TSDB_MAX_VNODES * (tsVnodePeers + 10) + tsSessionsPerVnode + 1000, 200, 60000, "DND-vnode"); + if (vnodeTmrCtrl == NULL) { + dError("failed to init timer, exit"); + return false; + } + return true; +} - rpcQhandle = malloc(tsMaxQueues*sizeof(void *)); - for (int i = 0; i < tsMaxQueues; i++) - rpcQhandle[i] = taosInitScheduler(tsSessionsPerVnode, 1, "dnode"); +int vnodeInitSystem() { - dmQhandle = taosInitScheduler(tsSessionsPerVnode, 1, "mgmt"); + if (!vnodeInitQueryHandle()) { + dError("failed to init query qhandle, exit"); + return -1; + } - vnodeTmrCtrl = taosTmrInit(tsSessionsPerVnode + 1000, 200, 60000, "DND-vnode"); - if (vnodeTmrCtrl == NULL) { + if (!vnodeInitTmrCtl()) { dError("failed to init timer, exit"); return -1; } @@ -62,6 +75,18 @@ int vnodeInitSystem() { return -1; } + int numOfThreads = (1.0 - tsRatioOfQueryThreads) * tsNumOfCores * tsNumOfThreadsPerCore / 2.0; + if (numOfThreads < 1) numOfThreads = 1; + if (vnodeInitPeer(numOfThreads) < 0) { + dError("failed to init vnode peer communication"); + return -1; + } + + if (vnodeInitMgmt() < 0) { + dError("failed to init communication to mgmt"); + return -1; + } + if (vnodeInitShell() < 0) { dError("failed to init communication to shell"); return -1; @@ -76,3 +101,15 @@ int vnodeInitSystem() { return 0; } + +void vnodeInitQHandle() { + tsMaxQueues = (1.0 - tsRatioOfQueryThreads)*tsNumOfCores*tsNumOfThreadsPerCore / 2.0; + if (tsMaxQueues < 1) tsMaxQueues = 1; + + rpcQhandle = malloc(tsMaxQueues*sizeof(void *)); + + for (int i=0; i< tsMaxQueues; ++i ) + rpcQhandle[i] = taosInitScheduler(tsSessionsPerVnode, 1, "dnode"); + + dmQhandle = taosInitScheduler(tsSessionsPerVnode, 1, "mgmt"); +} diff --git a/src/system/src/vnodeMeterTagMgmt.c b/src/system/detail/src/vnodeTagMgmt.c similarity index 81% rename from src/system/src/vnodeMeterTagMgmt.c rename to src/system/detail/src/vnodeTagMgmt.c index b31b34ae6a..9b76e14854 100644 --- a/src/system/src/vnodeMeterTagMgmt.c +++ b/src/system/detail/src/vnodeTagMgmt.c @@ -13,18 +13,16 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include #include #include "tsdb.h" - #include "tlog.h" #include "tutil.h" - #include "taosmsg.h" #include "textbuffer.h" - #include "tast.h" #include "vnodeTagMgmt.h" @@ -35,28 +33,21 @@ static void tTagsPrints(SMeterSidExtInfo *pMeterInfo, tTagSchema *pSchema, tOrde static void tSidSetDisplay(tSidSet *pSets); -// todo merge with losertree_compar/ext_comp -int32_t doCompare(char *f1, char *f2, int32_t type, int32_t size) { +//todo merge with losertree_compar/ext_comp +int32_t doCompare(char* f1, char* f2, int32_t type, int32_t size) { switch (type) { - case TSDB_DATA_TYPE_INT: - DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2)); - case TSDB_DATA_TYPE_DOUBLE: - DEFAULT_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); - case TSDB_DATA_TYPE_FLOAT: - DEFAULT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); - case TSDB_DATA_TYPE_BIGINT: - DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2)); - case TSDB_DATA_TYPE_SMALLINT: - DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2)); + case TSDB_DATA_TYPE_INT: DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2)); + case TSDB_DATA_TYPE_DOUBLE: DEFAULT_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); + case TSDB_DATA_TYPE_FLOAT: DEFAULT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); + case TSDB_DATA_TYPE_BIGINT: DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2)); + case TSDB_DATA_TYPE_SMALLINT: DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2)); case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_BOOL: - DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2)); + case TSDB_DATA_TYPE_BOOL: DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2)); case TSDB_DATA_TYPE_NCHAR: { - int32_t ret = wcsncmp((wchar_t *)f1, (wchar_t *)f2, size / TSDB_NCHAR_SIZE); + int32_t ret = wcsncmp((wchar_t*) f1, (wchar_t*) f2, size/TSDB_NCHAR_SIZE); if (ret == 0) { - return ret; + return ret; } - return (ret < 0) ? -1 : 1; } default: { @@ -190,12 +181,12 @@ void tQSortEx(void **pMeterSids, size_t size, int32_t start, int32_t end, void * if (ret == 0 && s != startLeftS) { tsDataSwap(&pMeterSids[s], &pMeterSids[startLeftS++], TSDB_DATA_TYPE_BINARY, size); } - s++; + s++; } - if (e != s) { - tsDataSwap(&pMeterSids[s], &pMeterSids[e], TSDB_DATA_TYPE_BINARY, size); - } + if (e != s) { + tsDataSwap(&pMeterSids[s], &pMeterSids[e], TSDB_DATA_TYPE_BINARY, size); + } } int32_t rightPartStart = e + 1; @@ -272,8 +263,8 @@ tTagSchema *tCreateTagSchema(SSchema *pSchema, int32_t numOfTagCols) { } tSidSet *tSidSetCreate(struct SMeterSidExtInfo **pMeterSidExtInfo, int32_t numOfMeters, SSchema *pSchema, - int32_t numOfTags, int16_t *orderList, int32_t numOfOrderCols) { - tSidSet *pSidSet = (tSidSet *)calloc(1, sizeof(tSidSet) + numOfOrderCols * sizeof(int16_t)); + int32_t numOfTags, SColIndexEx *colList, int32_t numOfCols) { + tSidSet *pSidSet = (tSidSet *)calloc(1, sizeof(tSidSet) + numOfCols * sizeof(int16_t)); if (pSidSet == NULL) { return NULL; } @@ -281,9 +272,19 @@ tSidSet *tSidSetCreate(struct SMeterSidExtInfo **pMeterSidExtInfo, int32_t numOf pSidSet->numOfSids = numOfMeters; pSidSet->pSids = pMeterSidExtInfo; pSidSet->pTagSchema = tCreateTagSchema(pSchema, numOfTags); - pSidSet->orderIdx.numOfOrderedCols = numOfOrderCols; + pSidSet->orderIdx.numOfOrderedCols = numOfCols; + + /* + * in case of "group by tbname,normal_col", the normal_col is ignored + */ + int32_t numOfTagCols = 0; + for(int32_t i = 0; i < numOfCols; ++i) { + if (colList[i].flag == TSDB_COL_TAG) { + pSidSet->orderIdx.pData[numOfTagCols++] = colList[i].colIdx; + } + } - memcpy(pSidSet->orderIdx.pData, orderList, numOfOrderCols * sizeof(int16_t)); + pSidSet->orderIdx.numOfOrderedCols = numOfTagCols; pSidSet->starterPos = NULL; return pSidSet; @@ -298,49 +299,53 @@ void tSidSetDestroy(tSidSet **pSets) { } void tTagsPrints(SMeterSidExtInfo *pMeterInfo, tTagSchema *pSchema, tOrderIdx *pOrder) { + if (pSchema == NULL) { + return; + } + printf("sid: %-5d tags(", pMeterInfo->sid); for (int32_t i = 0; i < pOrder->numOfOrderedCols; ++i) { - int32_t tagIdx = pOrder->pData[i]; + int32_t colIndex = pOrder->pData[i]; - if (tagIdx == -1) { - /* it is the tbname column */ + // it is the tbname column + if (colIndex == -1) { printf("%s, ", pMeterInfo->tags); continue; } - switch (pSchema->pSchema[tagIdx].type) { + switch (pSchema->pSchema[colIndex].type) { case TSDB_DATA_TYPE_INT: - printf("%d, ", GET_TAG_VAL(pMeterInfo, tagIdx, pSchema, int32_t)); + printf("%d, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, int32_t)); break; case TSDB_DATA_TYPE_DOUBLE: - printf("%lf, ", GET_TAG_VAL(pMeterInfo, tagIdx, pSchema, double)); + printf("%lf, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, double)); break; case TSDB_DATA_TYPE_FLOAT: - printf("%f, ", GET_TAG_VAL(pMeterInfo, tagIdx, pSchema, float)); + printf("%f, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, float)); break; case TSDB_DATA_TYPE_BIGINT: - printf("%ld, ", GET_TAG_VAL(pMeterInfo, tagIdx, pSchema, int64_t)); + printf("%ld, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, int64_t)); break; case TSDB_DATA_TYPE_SMALLINT: - printf("%d, ", GET_TAG_VAL(pMeterInfo, tagIdx, pSchema, int16_t)); + printf("%d, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, int16_t)); break; case TSDB_DATA_TYPE_TINYINT: - printf("%d, ", GET_TAG_VAL(pMeterInfo, tagIdx, pSchema, int8_t)); + printf("%d, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, int8_t)); break; case TSDB_DATA_TYPE_BINARY: - printf("%s, ", GET_TAG_VAL_POINTER(pMeterInfo, tagIdx, pSchema, char)); + printf("%s, ", GET_TAG_VAL_POINTER(pMeterInfo, colIndex, pSchema, char)); break; case TSDB_DATA_TYPE_NCHAR: { - char *data = GET_TAG_VAL_POINTER(pMeterInfo, tagIdx, pSchema, char); + char *data = GET_TAG_VAL_POINTER(pMeterInfo, colIndex, pSchema, char); char buffer[512] = {0}; - taosUcs4ToMbs(data, pSchema->pSchema[tagIdx].bytes, buffer); + taosUcs4ToMbs(data, pSchema->pSchema[colIndex].bytes, buffer); printf("%s, ", buffer); break; } case TSDB_DATA_TYPE_BOOL: - printf("%d, ", GET_TAG_VAL(pMeterInfo, tagIdx, pSchema, int8_t)); + printf("%d, ", GET_TAG_VAL(pMeterInfo, colIndex, pSchema, int8_t)); break; default: @@ -364,6 +369,7 @@ static void UNUSED_FUNC tSidSetDisplay(tSidSet *pSets) { for (int32_t i = 0; i < pSets->numOfSubSet; ++i) { int32_t s = pSets->starterPos[i]; int32_t e = pSets->starterPos[i + 1]; + printf("the %d-th subgroup: \n", i + 1); for (int32_t j = s; j < e; ++j) { tTagsPrints(pSets->pSids[j], pSets->pTagSchema, &pSets->orderIdx); @@ -375,13 +381,12 @@ void tSidSetSort(tSidSet *pSets) { pTrace("number of meters in sort: %d", pSets->numOfSids); tOrderIdx *pOrderIdx = &pSets->orderIdx; - if (pOrderIdx->numOfOrderedCols == 0 || pSets->numOfSids <= 1) { - // no group by clause + if (pOrderIdx->numOfOrderedCols == 0 || pSets->numOfSids <= 1 || pSets->pTagSchema == NULL) { // no group by tags clause pSets->numOfSubSet = 1; pSets->starterPos = (int32_t *)malloc(sizeof(int32_t) * (pSets->numOfSubSet + 1)); pSets->starterPos[0] = 0; pSets->starterPos[1] = pSets->numOfSids; - pTrace("all meters belong to one subgroup, no need to subgrouping ops."); + pTrace("all meters belong to one subgroup, no need to subgrouping ops"); #ifdef _DEBUG_VIEW tSidSetDisplay(pSets); #endif diff --git a/src/system/src/vnodeUtil.c b/src/system/detail/src/vnodeUtil.c similarity index 73% rename from src/system/src/vnodeUtil.c rename to src/system/detail/src/vnodeUtil.c index f8d50b6b81..790ad1c4a1 100644 --- a/src/system/src/vnodeUtil.c +++ b/src/system/detail/src/vnodeUtil.c @@ -13,12 +13,14 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include #include #include #include #include "tast.h" +#include "tscUtil.h" #include "tschemautil.h" #include "vnode.h" #include "vnodeDataFilterFunc.h" @@ -92,7 +94,7 @@ void vnodeCreateFileHeader(FILE* fp) { lineLen = sizeof(temp); - // write the first line` + // write the first line memset(temp, 0, lineLen); *(int16_t*)temp = vnodeFileVersion; sprintf(temp + sizeof(int16_t), "tsdb version: %s\n", version); @@ -111,30 +113,32 @@ void vnodeCreateFileHeader(FILE* fp) { } SSqlGroupbyExpr* vnodeCreateGroupbyExpr(SQueryMeterMsg* pQueryMsg, int32_t* code) { - if (pQueryMsg->numOfGroupbyCols == 0) { + if (pQueryMsg->numOfGroupCols == 0) { return NULL; } // using group by tag columns SSqlGroupbyExpr* pGroupbyExpr = - (SSqlGroupbyExpr*)malloc(sizeof(SSqlGroupbyExpr) + pQueryMsg->numOfGroupbyCols * sizeof(int16_t)); + (SSqlGroupbyExpr*)malloc(sizeof(SSqlGroupbyExpr) + pQueryMsg->numOfGroupCols * sizeof(SColIndexEx)); if (pGroupbyExpr == NULL) { *code = TSDB_CODE_SERV_OUT_OF_MEMORY; return NULL; } - int16_t* pGroupbyIds = (int16_t*)pQueryMsg->groupbyTagIds; + SColIndexEx* pGroupbyColInfo = (SColIndexEx*)pQueryMsg->groupbyTagIds; - pGroupbyExpr->numOfGroupbyCols = pQueryMsg->numOfGroupbyCols; + pGroupbyExpr->numOfGroupCols = pQueryMsg->numOfGroupCols; pGroupbyExpr->orderType = pQueryMsg->orderType; - pGroupbyExpr->orderIdx = pQueryMsg->orderByIdx; + pGroupbyExpr->orderIndex = pQueryMsg->orderByIdx; + + memcpy(pGroupbyExpr->columnInfo, pGroupbyColInfo, sizeof(SColIndexEx) * pGroupbyExpr->numOfGroupCols); - memcpy(pGroupbyExpr->tagIndex, pGroupbyIds, sizeof(int16_t) * pGroupbyExpr->numOfGroupbyCols); + // TODO: update the colIndexInBuf for each column in group by clause return pGroupbyExpr; } -static SSchema* toSchema(SQueryMeterMsg* pQuery, SColumnFilterMsg* pCols, int32_t numOfCols) { +static SSchema* toSchema(SQueryMeterMsg* pQuery, SColumnInfo* pCols, int32_t numOfCols) { char* start = (char*)pQuery->colNameList; char* end = start; @@ -158,7 +162,7 @@ static int32_t id_compar(const void* left, const void* right) { static int32_t vnodeBuildExprFromArithmeticStr(SSqlFunctionExpr* pExpr, SQueryMeterMsg* pQueryMsg) { SSqlBinaryExprInfo* pBinaryExprInfo = &pExpr->pBinExprInfo; - SColumnFilterMsg* pColMsg = pQueryMsg->colList; + SColumnInfo* pColMsg = pQueryMsg->colList; tSQLBinaryExpr* pBinExpr = NULL; SSchema* pSchema = toSchema(pQueryMsg, pColMsg, pQueryMsg->numOfCols); @@ -234,9 +238,12 @@ SSqlFunctionExpr* vnodeCreateSqlFunctionExpr(SQueryMeterMsg* pQueryMsg, int32_t* return NULL; } + bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType); + int16_t tagLen = 0; + SSchema* pTagSchema = (SSchema*)pQueryMsg->pTagSchema; for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { - pExprs[i].pBase = *((SSqlFuncExprMsg**)pQueryMsg->pSqlFuncExprs)[i]; // todo pExprs responsible for release memory + pExprs[i].pBase = *((SSqlFuncExprMsg**)pQueryMsg->pSqlFuncExprs)[i]; pExprs[i].resBytes = 0; int16_t type = 0; @@ -245,7 +252,7 @@ SSqlFunctionExpr* vnodeCreateSqlFunctionExpr(SQueryMeterMsg* pQueryMsg, int32_t* SColIndexEx* pColumnIndexExInfo = &pExprs[i].pBase.colInfo; // tag column schema is kept in pQueryMsg->pTagSchema - if (pColumnIndexExInfo->isTag) { + if (TSDB_COL_IS_TAG(pColumnIndexExInfo->flag)) { if (pColumnIndexExInfo->colIdx >= pQueryMsg->numOfTagsCols) { *code = TSDB_CODE_INVALID_QUERY_MSG; tfree(pExprs); @@ -270,16 +277,42 @@ SSqlFunctionExpr* vnodeCreateSqlFunctionExpr(SQueryMeterMsg* pQueryMsg, int32_t* int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase); assert(j < pQueryMsg->numOfCols); - SColumnFilterMsg* pCol = &pQueryMsg->colList[j]; + SColumnInfo* pCol = &pQueryMsg->colList[j]; type = pCol->type; bytes = pCol->bytes; } } int32_t param = pExprs[i].pBase.arg[0].argValue.i64; - getResultInfo(type, bytes, pExprs[i].pBase.functionId, param, &pExprs[i].resType, &pExprs[i].resBytes); + if (getResultDataInfo(type, bytes, pExprs[i].pBase.functionId, param, &pExprs[i].resType, &pExprs[i].resBytes, + &pExprs[i].interResBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) { + return NULL; + } + + if (pExprs[i].pBase.functionId == TSDB_FUNC_TAG_DUMMY) { + tagLen += pExprs[i].resBytes; + } + assert(isValidDataType(pExprs[i].resType, pExprs[i].resBytes)); + } + + //get the correct result size for top/bottom query, according to the number of tags columns in selection clause + + // TODO refactor + for(int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { + pExprs[i].pBase = *((SSqlFuncExprMsg**)pQueryMsg->pSqlFuncExprs)[i]; + int16_t functId = pExprs[i].pBase.functionId; + if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { + int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase); + assert(j < pQueryMsg->numOfCols); + + SColumnInfo* pCol = &pQueryMsg->colList[j]; + int16_t type = pCol->type; + int16_t bytes = pCol->bytes; - assert(pExprs[i].resType != 0 && pExprs[i].resBytes != 0); + int32_t ret = getResultDataInfo(type, bytes, pExprs[i].pBase.functionId, pExprs[i].pBase.arg[0].argValue.i64, + &pExprs[i].resType, &pExprs[i].resBytes, &pExprs[i].interResBytes, tagLen, isSuperTable); + assert(ret == TSDB_CODE_SUCCESS); + } } tfree(pQueryMsg->pSqlFuncExprs); @@ -289,7 +322,8 @@ SSqlFunctionExpr* vnodeCreateSqlFunctionExpr(SQueryMeterMsg* pQueryMsg, int32_t* bool vnodeIsValidVnodeCfg(SVnodeCfg* pCfg) { if (pCfg == NULL) return false; - if (pCfg->maxSessions <= 0 || pCfg->cacheBlockSize <= 0 || pCfg->daysPerFile <= 0 || pCfg->daysToKeep <= 0) { + if (pCfg->maxSessions <= 0 || pCfg->cacheBlockSize <= 0 || pCfg->replications <= 0 || pCfg->replications > 20 || + pCfg->daysPerFile <= 0 || pCfg->daysToKeep <= 0) { return false; } @@ -329,9 +363,9 @@ void vnodeFreeFields(SQuery* pQuery) { void vnodeUpdateFilterColumnIndex(SQuery* pQuery) { for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { for (int16_t j = 0; j < pQuery->numOfCols; ++j) { - if (pQuery->pFilterInfo[i].pFilter.data.colId == pQuery->colList[j].data.colId) { - pQuery->pFilterInfo[i].pFilter.colIdx = pQuery->colList[j].colIdx; - pQuery->pFilterInfo[i].pFilter.colIdxInBuf = pQuery->colList[j].colIdxInBuf; + if (pQuery->pFilterInfo[i].info.data.colId == pQuery->colList[j].data.colId) { + pQuery->pFilterInfo[i].info.colIdx = pQuery->colList[j].colIdx; + pQuery->pFilterInfo[i].info.colIdxInBuf = pQuery->colList[j].colIdxInBuf; // supplementary scan is also require this column pQuery->colList[j].req[1] = 1; @@ -361,9 +395,8 @@ void vnodeUpdateFilterColumnIndex(SQuery* pQuery) { // TODO support k<12 and k<>9 int32_t vnodeCreateFilterInfo(void* pQInfo, SQuery* pQuery) { - for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - if (pQuery->colList[i].data.filterOn > 0) { + if (pQuery->colList[i].data.numOfFilters > 0) { pQuery->numOfFilterCols++; } } @@ -372,56 +405,72 @@ int32_t vnodeCreateFilterInfo(void* pQInfo, SQuery* pQuery) { return TSDB_CODE_SUCCESS; } - pQuery->pFilterInfo = calloc(1, sizeof(SColumnFilterInfo) * pQuery->numOfFilterCols); + pQuery->pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * pQuery->numOfFilterCols); for (int32_t i = 0, j = 0; i < pQuery->numOfCols; ++i) { - if (pQuery->colList[i].data.filterOn > 0) { - pQuery->pFilterInfo[j].pFilter = pQuery->colList[i]; - SColumnFilterInfo* pFilterInfo = &pQuery->pFilterInfo[j]; + if (pQuery->colList[i].data.numOfFilters > 0) { + SSingleColumnFilterInfo* pFilterInfo = &pQuery->pFilterInfo[j]; - int32_t lower = pFilterInfo->pFilter.data.lowerRelOptr; - int32_t upper = pFilterInfo->pFilter.data.upperRelOptr; + memcpy(&pFilterInfo->info, &pQuery->colList[i], sizeof(SColumnInfoEx)); + pFilterInfo->info.data.filters = NULL; - int16_t type = pQuery->colList[i].data.type; - int16_t bytes = pQuery->colList[i].data.bytes; + pFilterInfo->numOfFilters = pQuery->colList[i].data.numOfFilters; + pFilterInfo->pFilters = calloc(pFilterInfo->numOfFilters, sizeof(SColumnFilterElem)); - __filter_func_t* rangeFilterArray = vnodeGetRangeFilterFuncArray(type); - __filter_func_t* filterArray = vnodeGetValueFilterFuncArray(type); + for(int32_t f = 0; f < pFilterInfo->numOfFilters; ++f) { + SColumnFilterElem *pSingleColFilter = &pFilterInfo->pFilters[f]; + pSingleColFilter->filterInfo = pQuery->colList[i].data.filters[f]; - if (rangeFilterArray == NULL && filterArray == NULL) { - dError("QInfo:%p failed to get filter function, invalid data type:%d", pQInfo, type); - return TSDB_CODE_INVALID_QUERY_MSG; - } + int32_t lower = pSingleColFilter->filterInfo.lowerRelOptr; + int32_t upper = pSingleColFilter->filterInfo.upperRelOptr; - if ((lower == TSDB_RELATION_LARGE_EQUAL || lower == TSDB_RELATION_LARGE) && - (upper == TSDB_RELATION_LESS_EQUAL || upper == TSDB_RELATION_LESS)) { - if (lower == TSDB_RELATION_LARGE_EQUAL) { - if (upper == TSDB_RELATION_LESS_EQUAL) { - pFilterInfo->fp = rangeFilterArray[4]; - } else { - pFilterInfo->fp = rangeFilterArray[2]; - } - } else { - if (upper == TSDB_RELATION_LESS_EQUAL) { - pFilterInfo->fp = rangeFilterArray[3]; + if (lower == TSDB_RELATION_INVALID && upper == TSDB_RELATION_INVALID) { + dError("QInfo:%p invalid filter info", pQInfo); + return TSDB_CODE_INVALID_QUERY_MSG; + } + + int16_t type = pQuery->colList[i].data.type; + int16_t bytes = pQuery->colList[i].data.bytes; + + __filter_func_t *rangeFilterArray = vnodeGetRangeFilterFuncArray(type); + __filter_func_t *filterArray = vnodeGetValueFilterFuncArray(type); + + if (rangeFilterArray == NULL && filterArray == NULL) { + dError("QInfo:%p failed to get filter function, invalid data type:%d", pQInfo, type); + return TSDB_CODE_INVALID_QUERY_MSG; + } + + if ((lower == TSDB_RELATION_LARGE_EQUAL || lower == TSDB_RELATION_LARGE) && + (upper == TSDB_RELATION_LESS_EQUAL || upper == TSDB_RELATION_LESS)) { + if (lower == TSDB_RELATION_LARGE_EQUAL) { + if (upper == TSDB_RELATION_LESS_EQUAL) { + pSingleColFilter->fp = rangeFilterArray[4]; + } else { + pSingleColFilter->fp = rangeFilterArray[2]; + } } else { - pFilterInfo->fp = rangeFilterArray[1]; + if (upper == TSDB_RELATION_LESS_EQUAL) { + pSingleColFilter->fp = rangeFilterArray[3]; + } else { + pSingleColFilter->fp = rangeFilterArray[1]; + } } - } - } else { // set callback filter function - if (lower != TSDB_RELATION_INVALID) { - pFilterInfo->fp = filterArray[lower]; + } else { // set callback filter function + if (lower != TSDB_RELATION_INVALID) { + pSingleColFilter->fp = filterArray[lower]; - if (upper != TSDB_RELATION_INVALID) { - dError("pQInfo:%p failed to get filter function, invalid filter condition", pQInfo, type); - return TSDB_CODE_INVALID_QUERY_MSG; + if (upper != TSDB_RELATION_INVALID) { + dError("pQInfo:%p failed to get filter function, invalid filter condition", pQInfo, type); + return TSDB_CODE_INVALID_QUERY_MSG; + } + } else { + pSingleColFilter->fp = filterArray[upper]; } - } else { - pFilterInfo->fp = filterArray[upper]; } + assert (pSingleColFilter->fp != NULL); + pSingleColFilter->bytes = bytes; } - pFilterInfo->elemSize = bytes; j++; } } @@ -431,14 +480,24 @@ int32_t vnodeCreateFilterInfo(void* pQInfo, SQuery* pQuery) { bool vnodeDoFilterData(SQuery* pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { - SColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - char* pElem = pFilterInfo->pData + pFilterInfo->elemSize * elemPos; + SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; + char* pElem = pFilterInfo->pData + pFilterInfo->info.data.bytes * elemPos; - if(isNull(pElem, pFilterInfo->pFilter.data.type)) { + if(isNull(pElem, pFilterInfo->info.data.type)) { return false; } - if (!pFilterInfo->fp(&pFilterInfo->pFilter, pElem, pElem)) { + int32_t num = pFilterInfo->numOfFilters; + bool qualified = false; + for(int32_t j = 0; j < num; ++j) { + SColumnFilterElem* pFilterElem = &pFilterInfo->pFilters[j]; + if (pFilterElem->fp(pFilterElem, pElem, pElem)) { + qualified = true; + break; + } + } + + if (!qualified) { return false; } } @@ -572,7 +631,7 @@ void vnodeUpdateQueryColumnIndex(SQuery* pQuery, SMeterObj* pMeterObj) { for(int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { SSqlFuncExprMsg* pSqlExprMsg = &pQuery->pSelectExpr[i].pBase; - if (pSqlExprMsg->functionId == TSDB_FUNC_ARITHM || pSqlExprMsg->colInfo.isTag == true) { + if (pSqlExprMsg->functionId == TSDB_FUNC_ARITHM || pSqlExprMsg->colInfo.flag == TSDB_COL_TAG) { continue; } @@ -648,3 +707,20 @@ bool vnodeIsSafeToDeleteMeter(SVnodeObj* pVnode, int32_t sid) { return ready; } + +void vnodeFreeColumnInfo(SColumnInfo* pColumnInfo) { + if (pColumnInfo == NULL) { + return; + } + + if (pColumnInfo->numOfFilters > 0) { + if (pColumnInfo->type == TSDB_DATA_TYPE_BINARY) { + for (int32_t i = 0; i < pColumnInfo->numOfFilters; ++i) { + tfree(pColumnInfo->filters[i].pz); + pColumnInfo->filters[i].len = 0; + } + } + + tfree(pColumnInfo->filters); + } +} diff --git a/src/system/lite/CMakeLists.txt b/src/system/lite/CMakeLists.txt new file mode 100644 index 0000000000..965f7666b0 --- /dev/null +++ b/src/system/lite/CMakeLists.txt @@ -0,0 +1,17 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +IF (TD_LINUX_64) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/modules/http/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/modules/monitor/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/system/detail/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/inc) + INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/util/cluster/inc) + INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) + INCLUDE_DIRECTORIES(inc) + + AUX_SOURCE_DIRECTORY(./src SRC) + ADD_LIBRARY(taosd_lite ${SRC}) +ENDIF () diff --git a/src/system/lite/src/dnodeMgmt.spec.c b/src/system/lite/src/dnodeMgmt.spec.c new file mode 100644 index 0000000000..00e7e469c0 --- /dev/null +++ b/src/system/lite/src/dnodeMgmt.spec.c @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "tsched.h" +#include "vnode.h" +#include "vnodeMgmt.h" + +void*vnodeProcessMsgFromMgmt(char *content, int msgLen, int msgType, SMgmtObj *pObj); +void mgmtProcessMsgFromDnodeSpec(SSchedMsg *sched); + +char *taosBuildRspMsgToMnodeWithSize(SMgmtObj *pObj, char type, int size) { + char *pStart = (char *)malloc(size); + if (pStart == NULL) { + return NULL; + } + + *pStart = type; + return pStart + 1; +} + +char *taosBuildReqMsgToMnodeWithSize(SMgmtObj *pObj, char type, int size) { + char *pStart = (char *)malloc(size); + if (pStart == NULL) { + return NULL; + } + + *pStart = type; + return pStart + 1; +} + +char *taosBuildRspMsgToMnode(SMgmtObj *pObj, char type) { + return taosBuildRspMsgToMnodeWithSize(pObj, type, 256); +} + +char *taosBuildReqMsgToMnode(SMgmtObj *pObj, char type) { + return taosBuildReqMsgToMnodeWithSize(pObj, type, 256); +} + +int taosSendMsgToMnode(SMgmtObj *pObj, char *msg, int msgLen) { + mTrace("msg:%s is sent to mnode", taosMsg[*(msg-1)]); + + /* + * Lite version has no message header, so minus one + */ + SSchedMsg schedMsg; + schedMsg.fp = mgmtProcessMsgFromDnodeSpec; + schedMsg.msg = msg - 1; + schedMsg.ahandle = NULL; + schedMsg.thandle = NULL; + taosScheduleTask(dmQhandle, &schedMsg); + + return 0; +} + +int taosSendSimpleRspToMnode(SMgmtObj *pObj, char rsptype, char code) { + char *pStart = taosBuildRspMsgToMnode(0, rsptype); + if (pStart == NULL) { + return 0; + } + + *pStart = code; + taosSendMsgToMnode(0, pStart, code); + + return 0; +} + +void vnodeProcessMsgFromMgmtSpec(SSchedMsg *sched) { + char msgType = *sched->msg; + char *content = sched->msg + 1; + + dTrace("msg:%s is received from mgmt", taosMsg[msgType]); + + vnodeProcessMsgFromMgmt(content, 0, msgType, 0); + + free(sched->msg); +} + +int vnodeInitMgmt() { return 0; } + +int vnodeSaveCreateMsgIntoQueue(SVnodeObj *pVnode, char *pMsg, int msgLen) { return 0; } \ No newline at end of file diff --git a/src/system/lite/src/dnodeSystem.spec.c b/src/system/lite/src/dnodeSystem.spec.c new file mode 100644 index 0000000000..7d6602c463 --- /dev/null +++ b/src/system/lite/src/dnodeSystem.spec.c @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mgmt.h" +#include "dnodeSystem.h" + +extern SModule tsModule[TSDB_MOD_MAX]; + +int taosCreateTierDirectory() { + struct stat dirstat; + strcpy(tsDirectory, dataDir); + if (stat(dataDir, &dirstat) < 0) { + mkdir(dataDir, 0755); + } + + char fileName[128]; + + sprintf(fileName, "%s/tsdb", tsDirectory); + mkdir(fileName, 0755); + + sprintf(fileName, "%s/data", tsDirectory); + mkdir(fileName, 0755); + + sprintf(mgmtDirectory, "%s/mgmt", tsDirectory); + sprintf(tsDirectory, "%s/tsdb", dataDir); + dnodeCheckDbRunning(dataDir); + + return 0; +} + +int dnodeInitSystemSpec() { return 0; } + +void dnodeStartModuleSpec() { + for (int mod = 1; mod < TSDB_MOD_MAX; ++mod) { + if (tsModule[mod].num != 0 && tsModule[mod].startFp) { + if ((*tsModule[mod].startFp)() != 0) { + dError("failed to start module:%d", mod); + } + } + } +} + +void dnodeParseParameterK() {} \ No newline at end of file diff --git a/src/system/lite/src/mgmtAcct.spec.c b/src/system/lite/src/mgmtAcct.spec.c new file mode 100644 index 0000000000..81f83df988 --- /dev/null +++ b/src/system/lite/src/mgmtAcct.spec.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mgmt.h" + +extern void *userSdb; +extern void *dbSdb; +SAcctObj acctObj; + +int mgmtInitAccts() { return 0; } + +void mgmtCreateRootAcct() {} + +SAcctObj *mgmtGetAcct(char *name) { return &acctObj; } + +int mgmtCheckUserLimit(SAcctObj *pAcct) { + int numOfUsers = sdbGetNumOfRows(userSdb); + if (numOfUsers >= tsMaxUsers) { + mWarn("numOfUsers:%d, exceed tsMaxUsers:%d", numOfUsers, tsMaxUsers); + return TSDB_CODE_TOO_MANY_USERS; + } + return 0; +} + +int mgmtCheckDbLimit(SAcctObj *pAcct) { + int numOfDbs = sdbGetNumOfRows(dbSdb); + if (numOfDbs >= tsMaxDbs) { + mWarn("numOfDbs:%d, exceed tsMaxDbs:%d", numOfDbs, tsMaxDbs); + return TSDB_CODE_TOO_MANY_DATABSES; + } + return 0; +} + +int mgmtCheckMeterLimit(SAcctObj *pAcct) { return 0; } + +int mgmtCheckUserGrant() { return 0; } + +int mgmtCheckDbGrant() { return 0; } + +int mgmtCheckMeterGrant() { return 0; } + +void grantAddTimeSeries(uint32_t timeSeriesNum) {} + +void mgmtCheckAcct() { + SAcctObj *pAcct = &acctObj; + pAcct->acctId = 0; + strcpy(pAcct->user, "root"); + + mgmtCreateUser(pAcct, "root", "taosdata"); + mgmtCreateUser(pAcct, "monitor", tsInternalPass); + mgmtCreateUser(pAcct, "_root", tsInternalPass); +} + +void mgmtCleanUpAccts() {} + +int mgmtGetAcctMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { return TSDB_CODE_OPS_NOT_SUPPORT; } + +int mgmtRetrieveAccts(SShowObj *pShow, char *data, int rows, SConnObj *pConn) { return 0; } diff --git a/src/system/lite/src/mgmtBalance.spec.c b/src/system/lite/src/mgmtBalance.spec.c new file mode 100644 index 0000000000..5100aea936 --- /dev/null +++ b/src/system/lite/src/mgmtBalance.spec.c @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mgmtBalance.h" + +void mgmtStartBalanceTimer(int mseconds) {} + +int mgmtInitBalance() { return 0; } + +void mgmtCleanupBalance() {} + +int mgmtAllocVnodes(SVgObj *pVgroup) { + int selectedVnode = -1; + SDnodeObj *pDnode = &dnodeObj; + + for (int i = 0; i < pDnode->numOfVnodes; i++) { + int vnode = (i + pDnode->lastAllocVnode) % pDnode->numOfVnodes; + if (pDnode->vload[vnode].vgId == 0 && pDnode->vload[vnode].status == TSDB_VN_STATUS_READY) { + selectedVnode = vnode; + break; + } + } + + if (selectedVnode == -1) { + mError("vgroup:%d alloc vnode failed, free vnodes:%d", pVgroup->vgId, pDnode->numOfFreeVnodes); + return -1; + } else { + mTrace("vgroup:%d allocate vnode:%d, last allocated vnode:%d", pVgroup->vgId, selectedVnode, + pDnode->lastAllocVnode); + pVgroup->vnodeGid[0].vnode = selectedVnode; + pDnode->lastAllocVnode = selectedVnode + 1; + if (pDnode->lastAllocVnode >= pDnode->numOfVnodes) pDnode->lastAllocVnode = 0; + return 0; + } +} + +bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int moduleType) { + return tsModule[moduleType].num != 0; +} + +bool mgmtCheckVnodeReady(SDnodeObj *pDnode, SVgObj *pVgroup, SVnodeGid *pVnode) { return true; } + +void mgmtUpdateDnodeState(SDnodeObj *pDnode, int lbState) {} + +void mgmtUpdateVgroupState(SVgObj *pVgroup, int lbState, int srcIp) {} + +bool mgmtAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) { return false; } \ No newline at end of file diff --git a/src/system/lite/src/mgmtDnode.spec.c b/src/system/lite/src/mgmtDnode.spec.c new file mode 100644 index 0000000000..c34ac58c00 --- /dev/null +++ b/src/system/lite/src/mgmtDnode.spec.c @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mgmt.h" + +SDnodeObj dnodeObj; +extern uint32_t tsRebootTime; + +SDnodeObj *mgmtGetDnode(uint32_t ip) { return &dnodeObj; } + +int mgmtUpdateDnode(SDnodeObj *pDnode) { return 0; } + +void mgmtCleanUpDnodes() {} + +int mgmtInitDnodes() { + dnodeObj.privateIp = inet_addr(tsInternalIp);; + dnodeObj.createdTime = (int64_t)tsRebootTime * 1000; + dnodeObj.lastReboot = tsRebootTime; + dnodeObj.numOfCores = (uint16_t)tsNumOfCores; + dnodeObj.status = TSDB_STATUS_READY; + dnodeObj.alternativeRole = TSDB_DNODE_ROLE_ANY; + dnodeObj.numOfTotalVnodes = tsNumOfTotalVnodes; + dnodeObj.thandle = (void*)(1); //hack way + if (dnodeObj.numOfVnodes == TSDB_INVALID_VNODE_NUM) { + mgmtSetDnodeMaxVnodes(&dnodeObj); + mPrint("first access, set total vnodes:%d", dnodeObj.numOfVnodes); + } + return 0; +} + +int mgmtGetDnodesNum() { return 1; } + +void *mgmtGetNextDnode(SShowObj *pShow, SDnodeObj **pDnode) { + if (*pDnode == NULL) { + *pDnode = &dnodeObj; + } else { + *pDnode = NULL; + } + + return *pDnode; +} + +int mgmtGetScoresMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { return TSDB_CODE_OPS_NOT_SUPPORT; } + +int mgmtRetrieveScores(SShowObj *pShow, char *data, int rows, SConnObj *pConn) { return 0; } + +void mgmtSetDnodeUnRemove(SDnodeObj *pDnode) {} + +bool mgmtCheckConfigShow(SGlobalConfig *cfg) { + if (cfg->cfgType & TSDB_CFG_CTYPE_B_CLUSTER) + return false; + if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) + return false; + return true; +} \ No newline at end of file diff --git a/src/system/src/mgmtSystem.c b/src/system/lite/src/mgmtDnodeInt.spec.c similarity index 52% rename from src/system/src/mgmtSystem.c rename to src/system/lite/src/mgmtDnodeInt.spec.c index acaa82c831..acde36e7b8 100644 --- a/src/system/src/mgmtSystem.c +++ b/src/system/lite/src/mgmtDnodeInt.spec.c @@ -13,47 +13,71 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include "dnodeSystem.h" #include "mgmt.h" -#include "tsdb.h" -#include "tsystem.h" +#include "tsched.h" +#include "tutil.h" #include "vnode.h" +#include "tsystem.h" + +extern void *dmQhandle; +void * mgmtStatusTimer = NULL; +void mgmtProcessMsgFromDnode(char *content, int msgLen, int msgType, SDnodeObj *pObj); +void vnodeProcessMsgFromMgmtSpec(SSchedMsg *sched); + +char *taosBuildRspMsgToDnodeWithSize(SDnodeObj *pObj, char type, int size) { + char *pStart = (char *)malloc(size); + if (pStart == NULL) { + return NULL; + } + + *pStart = type; + return pStart + 1; +} + +char *taosBuildReqMsgToDnodeWithSize(SDnodeObj *pObj, char type, int size) { + char *pStart = (char *)malloc(size); + if (pStart == NULL) { + return NULL; + } + + *pStart = type; + return pStart + 1; +} -// global, not configurable -char mgmtDirectory[128]; -void * mgmtTmr; -void * mgmtQhandle; -void * mgmtTranQhandle = NULL; -void * mgmtStatisticTimer = NULL; -void * mgmtStatusTimer = NULL; -int mgmtShellConns = 0; -extern void *pShellConn; - -void mgmtCleanUpSystem() { - mTrace("mgmt is running, clean it up"); - taosTmrStopA(&mgmtStatisticTimer); - mgmtCleanUpShell(); - mgmtCleanUpMeters(); - mgmtCleanUpVgroups(); - mgmtCleanUpDbs(); - mgmtCleanUpUsers(); - taosTmrCleanUp(mgmtTmr); - taosCleanUpScheduler(mgmtQhandle); - taosCleanUpScheduler(mgmtTranQhandle); +char *taosBuildRspMsgToDnode(SDnodeObj *pObj, char type) { + return taosBuildRspMsgToDnodeWithSize(pObj, type, 256); } -void mgmtDoStatistic(void *handle, void *tmrId) {} +char *taosBuildReqMsgToDnode(SDnodeObj *pObj, char type) { + return taosBuildReqMsgToDnodeWithSize(pObj, type, 256); +} + +int taosSendSimpleRspToDnode(SDnodeObj *pObj, char rsptype, char code) { return 0; } + +int taosSendMsgToDnode(SDnodeObj *pObj, char *msg, int msgLen) { + mTrace("msg:%s is sent to dnode", taosMsg[*(msg-1)]); + + /* + * Lite version has no message header, so minus one + */ + SSchedMsg schedMsg; + schedMsg.fp = vnodeProcessMsgFromMgmtSpec; + schedMsg.msg = msg - 1; + schedMsg.ahandle = NULL; + schedMsg.thandle = NULL; + taosScheduleTask(dmQhandle, &schedMsg); + + return 0; +} + +int mgmtInitDnodeInt() { return 0; } + +void mgmtCleanUpDnodeInt() {} void mgmtProcessDnodeStatus(void *handle, void *tmrId) { SDnodeObj *pObj = &dnodeObj; @@ -62,7 +86,6 @@ void mgmtProcessDnodeStatus(void *handle, void *tmrId) { float memoryUsedMB = 0; taosGetSysMemory(&memoryUsedMB); - pObj->memoryAvailable = tsTotalMemoryMB - memoryUsedMB; pObj->diskAvailable = tsAvailDataDirGB; for (int vnode = 0; vnode < pObj->numOfVnodes; ++vnode) { @@ -115,65 +138,11 @@ void mgmtProcessDnodeStatus(void *handle, void *tmrId) { } } -int mgmtInitSystem() { - mPrint("starting to initialize TDengine mgmt ..."); - - struct stat dirstat; - - if (stat(mgmtDirectory, &dirstat) < 0) mkdir(mgmtDirectory, 0755); +void mgmtProcessMsgFromDnodeSpec(SSchedMsg *sched) { + char msgType = *sched->msg; + char *content = sched->msg + 1; + mTrace("msg:%s is received from dnode", taosMsg[msgType]); - int numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore / 2.0; - if (numOfThreads < 1) numOfThreads = 1; - mgmtQhandle = taosInitScheduler(tsMaxDnodes + tsMaxShellConns, numOfThreads, "mnode"); - - mgmtTranQhandle = taosInitScheduler(tsMaxDnodes + tsMaxShellConns, 1, "mnodeT"); - - mgmtTmr = taosTmrInit((tsMaxDnodes + tsMaxShellConns) * 3, 200, 3600000, "MND"); - if (mgmtTmr == NULL) { - mError("failed to init timer, exit"); - return -1; - } - - dnodeObj.lastReboot = tsRebootTime; - dnodeObj.numOfCores = (uint16_t)tsNumOfCores; - dnodeObj.status = TSDB_STATUS_READY; - if (dnodeObj.numOfVnodes == TSDB_INVALID_VNODE_NUM) { - mgmtSetDnodeMaxVnodes(&dnodeObj); - mPrint("first access, set total vnodes:%d", dnodeObj.numOfVnodes); - } - - if (mgmtInitUsers() < 0) { - mError("failed to init users"); - return -1; - } - - if (mgmtInitDbs() < 0) { - mError("failed to init dbs"); - return -1; - } - - if (mgmtInitVgroups() < 0) { - mError("failed to init vgroups"); - return -1; - } - - if (mgmtInitMeters() < 0) { - mError("failed to init meters"); - return -1; - } - - if (mgmtInitShell() < 0) { - mError("failed to init shell"); - return -1; - } - - mgmtCheckAcct(); - - taosTmrReset(mgmtDoStatistic, tsStatusInterval * 30000, NULL, mgmtTmr, &mgmtStatisticTimer); - - taosTmrReset(mgmtProcessDnodeStatus, 500, NULL, mgmtTmr, &mgmtStatusTimer); - - mPrint("TDengine mgmt is initialized successfully"); - - return 0; + mgmtProcessMsgFromDnode(content, 0, msgType, mgmtGetDnode(0)); + free(sched->msg); } diff --git a/src/system/lite/src/mgmtMnode.spec.c b/src/system/lite/src/mgmtMnode.spec.c new file mode 100644 index 0000000000..e24f38adf2 --- /dev/null +++ b/src/system/lite/src/mgmtMnode.spec.c @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mgmt.h" + +int mgmtGetMnodeMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { return TSDB_CODE_OPS_NOT_SUPPORT; } + +int mgmtRetrieveMnodes(SShowObj *pShow, char *data, int rows, SConnObj *pConn) { return 0; } diff --git a/src/system/lite/src/mgmtShell.spec.c b/src/system/lite/src/mgmtShell.spec.c new file mode 100644 index 0000000000..5195010b41 --- /dev/null +++ b/src/system/lite/src/mgmtShell.spec.c @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include +#include "mgmt.h" + +int mgmtCheckRedirectMsg(SConnObj *pConn, int msgType) { return 0; } + +int mgmtProcessAlterAcctMsg(char *pMsg, int msgLen, SConnObj *pConn) { + return taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_ALTER_ACCT_RSP, TSDB_CODE_OPS_NOT_SUPPORT); +} + +int mgmtProcessCreateDnodeMsg(char *pMsg, int msgLen, SConnObj *pConn) { + return taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_CREATE_PNODE_RSP, TSDB_CODE_OPS_NOT_SUPPORT); +} + +int mgmtProcessCfgMnodeMsg(char *pMsg, int msgLen, SConnObj *pConn) { + return taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_CFG_MNODE_RSP, TSDB_CODE_OPS_NOT_SUPPORT); +} + +int mgmtProcessDropMnodeMsg(char *pMsg, int msgLen, SConnObj *pConn) { + return taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_DROP_MNODE_RSP, TSDB_CODE_OPS_NOT_SUPPORT); +} + +int mgmtProcessDropDnodeMsg(char *pMsg, int msgLen, SConnObj *pConn) { + return taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_DROP_PNODE_RSP, TSDB_CODE_OPS_NOT_SUPPORT); +} + +int mgmtProcessDropAcctMsg(char *pMsg, int msgLen, SConnObj *pConn) { + return taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_DROP_ACCT_RSP, TSDB_CODE_OPS_NOT_SUPPORT); +} + +int mgmtProcessCreateAcctMsg(char *pMsg, int msgLen, SConnObj *pConn) { + return taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_CREATE_ACCT_RSP, TSDB_CODE_OPS_NOT_SUPPORT); +} \ No newline at end of file diff --git a/src/system/lite/src/mgmtSystem.spec.c b/src/system/lite/src/mgmtSystem.spec.c new file mode 100644 index 0000000000..47e36bf1a4 --- /dev/null +++ b/src/system/lite/src/mgmtSystem.spec.c @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include + +#include "dnodeSystem.h" +#include "mgmt.h" + +extern void *mgmtTmr; +extern void *mgmtStatusTimer; + +void mgmtProcessDnodeStatus(void *handle, void *tmrId); + +int mgmtInitSystem() { return mgmtStartSystem(); } + +int32_t mgmtStartCheckMgmtRunning() { return 0; } + +void mgmtDoStatistic(void *handle, void *tmrId) {} + +void mgmtStartMgmtTimer() { taosTmrReset(mgmtProcessDnodeStatus, 500, NULL, mgmtTmr, &mgmtStatusTimer); } + +void mgmtStopSystem() {} + +void mgmtCleanUpRedirect() {} + +bool grantCheckExpired() { return false; } + +int grantGetGrantsMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { return TSDB_CODE_OPS_NOT_SUPPORT; } + +int grantRetrieveGrants(SShowObj *pShow, char *data, int rows, SConnObj *pConn) { return 0; } + +void grantRestoreTimeSeries(uint32_t timeseries) {} \ No newline at end of file diff --git a/src/system/lite/src/vnodeFile.spec.c b/src/system/lite/src/vnodeFile.spec.c new file mode 100644 index 0000000000..53651a8cc9 --- /dev/null +++ b/src/system/lite/src/vnodeFile.spec.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "vnode.h" +#include "vnodeFile.h" + +char* vnodeGetDiskFromHeadFile(char *headName) { return tsDirectory; } + +char* vnodeGetDataDir(int vnode, int fileId) { return dataDir; } + +void vnodeAdustVnodeFile(SVnodeObj *pVnode) { + // Retention policy here + int fileId = pVnode->fileId - pVnode->numOfFiles + 1; + int cfile = taosGetTimestamp(pVnode->cfg.precision)/pVnode->cfg.daysPerFile/tsMsPerDay[pVnode->cfg.precision]; + while (fileId <= cfile - pVnode->maxFiles) { + vnodeRemoveFile(pVnode->vnode, fileId); + pVnode->numOfFiles--; + fileId++; + } +} + +int vnodeCheckNewHeaderFile(int fd, SVnodeObj *pVnode) { + SCompHeader *pHeader = NULL; + SCompBlock *pBlocks = NULL; + int blockSize = 0; + SCompInfo compInfo; + int tmsize = 0; + + tmsize = sizeof(SCompHeader) * pVnode->cfg.maxSessions + sizeof(TSCKSUM); + + pHeader = (SCompHeader *)malloc(tmsize); + if (pHeader == NULL) return 0; + + lseek(fd, TSDB_FILE_HEADER_LEN, SEEK_SET); + if (read(fd, (void *)pHeader, tmsize) != tmsize) { + goto _broken_exit; + } + + if (!taosCheckChecksumWhole((uint8_t *)pHeader, tmsize)) { + goto _broken_exit; + } + + for (int sid = 0; sid < pVnode->cfg.maxSessions; sid++) { + if (pVnode->meterList == NULL) goto _correct_exit; + if (pVnode->meterList[sid] == NULL || pHeader[sid].compInfoOffset == 0) continue; + lseek(fd, pHeader[sid].compInfoOffset, SEEK_SET); + + if (read(fd, (void *)(&compInfo), sizeof(SCompInfo)) != sizeof(SCompInfo)) { + goto _broken_exit; + } + + if (!taosCheckChecksumWhole((uint8_t *)(&compInfo), sizeof(SCompInfo))) { + goto _broken_exit; + } + + if (compInfo.uid != ((SMeterObj *)pVnode->meterList[sid])->uid) continue; + + int expectedSize = sizeof(SCompBlock) * compInfo.numOfBlocks + sizeof(TSCKSUM); + if (blockSize < expectedSize) { + pBlocks = (SCompBlock *)realloc(pBlocks, expectedSize); + if (pBlocks == NULL) { + tfree(pHeader); + return 0; + } + + blockSize = expectedSize; + } + + if (read(fd, (void *)pBlocks, expectedSize) != expectedSize) { + dError("failed to read block part"); + goto _broken_exit; + } + if (!taosCheckChecksumWhole((uint8_t *)pBlocks, expectedSize)) { + dError("block part is broken"); + goto _broken_exit; + } + + for (int i = 0; i < compInfo.numOfBlocks; i++) { + if (pBlocks[i].last && i != compInfo.numOfBlocks-1) { + dError("last block in middle, block:%d", i); + goto _broken_exit; + } + } + } + + _correct_exit: + dTrace("vid: %d new header file %s is correct", pVnode->vnode, pVnode->nfn); + tfree(pBlocks); + tfree(pHeader); + return 0; + + _broken_exit: + dError("vid: %d new header file %s is broken", pVnode->vnode, pVnode->nfn); + tfree(pBlocks); + tfree(pHeader); + return -1; +} \ No newline at end of file diff --git a/src/system/lite/src/vnodePeer.spec.c b/src/system/lite/src/vnodePeer.spec.c new file mode 100644 index 0000000000..1ceb8465c3 --- /dev/null +++ b/src/system/lite/src/vnodePeer.spec.c @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "vnode.h" + +int vnodeInitPeer(int numOfThreads) { return 0; } + +void vnodeCleanUpPeer(int vnode) {} + +int vnodeForwardToPeer(SMeterObj *pObj, char *cont, int contLen, char action, int sversion) { return 0; } + +int vnodeRecoverFromPeer(SVnodeObj *pVnode, int fileId) { return -TSDB_CODE_FILE_CORRUPTED; } + +void vnodeCloseAllSyncFds(int vnode) {} + +void vnodeBroadcastStatusToUnsyncedPeer(SVnodeObj *pVnode) {} + +int vnodeOpenPeerVnode(int vnode) { + SVnodeObj *pVnode = vnodeList + vnode; + pVnode->status = (pVnode->cfg.replications > 1) ? TSDB_STATUS_UNSYNCED : TSDB_STATUS_MASTER; + dTrace("vid:%d, vnode status:%d numOfPeers:%d", vnode, pVnode->status, pVnode->cfg.replications-1); + vnodeUpdateStreamRole(pVnode); + return 0; +} + +void vnodeClosePeerVnode(int vnode) {} + +void vnodeConfigVPeers(int vnode, int numOfPeers, SVPeerDesc peerDesc[]) {} \ No newline at end of file diff --git a/src/system/lite/src/vnodeStore.spec.c b/src/system/lite/src/vnodeStore.spec.c new file mode 100644 index 0000000000..2259d1267e --- /dev/null +++ b/src/system/lite/src/vnodeStore.spec.c @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include + +int vnodeInitInfo() { return 0; } + +bool vnodeRemoveDataFileFromLinkFile(char* linkFile, char* de_name) { return true; } + diff --git a/src/system/src/mgmtDnode.c b/src/system/src/mgmtDnode.c deleted file mode 100644 index a852c888a9..0000000000 --- a/src/system/src/mgmtDnode.c +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE - -#include -#include -#include - -#include "dnodeSystem.h" -#include "mgmt.h" -#include "tschemautil.h" -#include "tstatus.h" -#pragma GCC diagnostic ignored "-Wunused-variable" - -SDnodeObj dnodeObj; - -void mgmtSetDnodeMaxVnodes(SDnodeObj *pDnode) { - int maxVnodes = pDnode->numOfCores * tsNumOfVnodesPerCore; - maxVnodes = maxVnodes > TSDB_MAX_VNODES ? TSDB_MAX_VNODES : maxVnodes; - maxVnodes = maxVnodes < TSDB_MIN_VNODES ? TSDB_MIN_VNODES : maxVnodes; - if (tsNumOfTotalVnodes != 0) { - maxVnodes = tsNumOfTotalVnodes; - } - if (pDnode->alternativeRole == TSDB_DNODE_ROLE_MGMT) { - maxVnodes = 0; - } - - pDnode->numOfVnodes = maxVnodes; - pDnode->numOfFreeVnodes = maxVnodes; - pDnode->openVnodes = 0; -} - -void mgmtCalcNumOfFreeVnodes(SDnodeObj *pDnode) { - int totalVnodes = 0; - - for (int i = 0; i < pDnode->numOfVnodes; ++i) { - SVnodeLoad *pVload = pDnode->vload + i; - if (pVload->vgId != 0) { - totalVnodes++; - } - } - - pDnode->numOfFreeVnodes = pDnode->numOfVnodes - totalVnodes; -} - -void mgmtSetDnodeVgid(int vnode, int vgId) { - SDnodeObj *pDnode = &dnodeObj; - - SVnodeLoad *pVload = pDnode->vload + vnode; - memset(pVload, 0, sizeof(SVnodeLoad)); - pVload->vnode = vnode; - pVload->vgId = vgId; - mgmtCalcNumOfFreeVnodes(pDnode); -} - -void mgmtUnSetDnodeVgid(int vnode) { - SDnodeObj *pDnode = &dnodeObj; - - SVnodeLoad *pVload = pDnode->vload + vnode; - memset(pVload, 0, sizeof(SVnodeLoad)); - mgmtCalcNumOfFreeVnodes(pDnode); -} - -int mgmtGetDnodeMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) { - int cols = 0; - - SSchema *pSchema = tsGetSchema(pMeta); - - pShow->bytes[cols] = 2; - pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "open vnodes"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 2; - pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "free vnodes"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pMeta->numOfColumns = htons(cols); - pShow->numOfColumns = cols; - - pShow->offset[0] = 0; - for (int i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; - - pShow->numOfRows = 1; - pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - pShow->pNode = NULL; - - return 0; -} - -int mgmtRetrieveDnodes(SShowObj *pShow, char *data, int rows, SConnObj *pConn) { - int numOfRows = 0; - SDnodeObj *pDnode = &dnodeObj; - char * pWrite; - int cols = 0; - char ipstr[20]; - - if (pShow->numOfReads > 0) return 0; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pDnode->openVnodes; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pDnode->numOfFreeVnodes; - cols++; - - pShow->numOfReads += 1; - return 1; -} \ No newline at end of file diff --git a/src/system/src/mgmtDnodeInt.c b/src/system/src/mgmtDnodeInt.c deleted file mode 100644 index 8be91fd519..0000000000 --- a/src/system/src/mgmtDnodeInt.c +++ /dev/null @@ -1,340 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include -#include - -#include "dnodeSystem.h" -#include "mgmt.h" -#include "tsched.h" -#include "tutil.h" -#pragma GCC diagnostic ignored "-Wpointer-sign" - -int mgmtSendVPeersMsg(SVgObj *pVgroup, SDbObj *pDb); -char *mgmtBuildVpeersIe(char *pMsg, SVgObj *pVgroup, SDbObj *pDb); -char *mgmtBuildCreateMeterIe(STabObj *pMeter, char *pMsg, int vnode); - -void vnodeProcessMsgFromMgmt(SSchedMsg *smsg); -void *rpcQhandle; -extern void *dmQhandle; - -int mgmtSendMsgToDnode(char *msg) { - mTrace("msg:%s is sent to dnode", taosMsg[*msg]); - - SSchedMsg schedMsg; - schedMsg.fp = vnodeProcessMsgFromMgmt; - schedMsg.msg = msg; - schedMsg.ahandle = NULL; - schedMsg.thandle = NULL; - taosScheduleTask(dmQhandle, &schedMsg); - - return 0; -} - -int mgmtProcessMeterCfgMsg(unsigned char *cont) { - char * pMsg, *pStart; - STabObj * pMeter = NULL; - SMeterCfgMsg *pCfg = (SMeterCfgMsg *)cont; - SVgObj * pVgroup; - SDnodeObj * pObj = &dnodeObj; - - int vnode = htonl(pCfg->vnode); - int sid = htonl(pCfg->sid); - - pStart = (char *)malloc(64000); - if (pStart == NULL) return 0; - - *pStart = TSDB_MSG_TYPE_METER_CFG_RSP; - pMsg = pStart + 1; - - if (vnode < pObj->numOfVnodes) { - int vgId = pObj->vload[vnode].vgId; - - pVgroup = mgmtGetVgroup(vgId); - if (pVgroup) pMeter = pVgroup->meterList[sid]; - } - - if (pMeter) { - *pMsg = 0; // code - pMsg++; - pMsg = mgmtBuildCreateMeterIe(pMeter, pMsg, vnode); - } else { - mTrace("vnode:%d sid:%d, meter not there", vnode, sid); - *pMsg = TSDB_CODE_INVALID_METER_ID; - pMsg++; - - *(int32_t *)pMsg = htonl(vnode); - pMsg += sizeof(int32_t); - *(int32_t *)pMsg = htonl(sid); - pMsg += sizeof(int32_t); - } - - mgmtSendMsgToDnode(pStart); - - return 0; -} - -int mgmtProcessVpeerCfgMsg(unsigned char *cont) { - char * pMsg, *pStart; - SVpeerCfgMsg *pCfg = (SVpeerCfgMsg *)cont; - SVgObj * pVgroup = NULL; - SDnodeObj * pObj = &dnodeObj; - - int vnode = htonl(pCfg->vnode); - - pStart = (char *)malloc(256); - if (pStart == NULL) return 0; - - *pStart = TSDB_MSG_TYPE_VPEER_CFG_RSP; - pMsg = pStart + 1; - - if (vnode < pObj->numOfVnodes) pVgroup = mgmtGetVgroup(pObj->vload[vnode].vgId); - - if (pVgroup) { - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - *pMsg = 0; - pMsg++; - pMsg = mgmtBuildVpeersIe(pMsg, pVgroup, pDb); - } else { - mTrace("vnode:%d, no vgroup info, vgroup:%d", vnode, pObj->vload[vnode].vgId); - *pMsg = TSDB_CODE_INVALID_VALUE; - pMsg++; - *(int32_t *)pMsg = htonl(vnode); - pMsg += sizeof(int32_t); - } - - return 0; -} - -int mgmtProcessCreateRsp(unsigned char *msg) { return 0; } - -int mgmtProcessFreeVnodeRsp(unsigned char *msg) { return 0; } - -int mgmtProcessVPeersRsp(unsigned char *msg) { - STaosRsp *pRsp = (STaosRsp *)msg; - - SDbObj *pDb = mgmtGetDb(pRsp->more); - if (!pDb) { - mError("db not find, code:%d", pRsp->code); - return 0; - } - - if (pDb->vgStatus != TSDB_VG_STATUS_IN_PROGRESS) { - mTrace("db:%s vpeer rsp already disposed, code:%d", pRsp->more, pRsp->code); - return 0; - } - - if (pRsp->code == 0) { - pDb->vgStatus = TSDB_VG_STATUS_READY; - mTrace("db:%s vgroup is created in dnode", pRsp->more); - return 0; - } - - if (pRsp->code == TSDB_CODE_VG_COMMITLOG_INIT_FAILED) { - pDb->vgStatus = TSDB_VG_STATUS_COMMITLOG_INIT_FAILED; - } else { - pDb->vgStatus = TSDB_VG_STATUS_INIT_FAILED; - } - mError("db:%s vgroup create failed, code:%d", pRsp->more, pRsp->code); - - return 0; -} - -void mgmtProcessMsgFromVnode(SSchedMsg *sched) { - char msgType = *sched->msg; - char *content = sched->msg + 1; - - mTrace("msg:%s is received from dnode", taosMsg[msgType]); - - if (msgType == TSDB_MSG_TYPE_METER_CFG) { - mgmtProcessMeterCfgMsg(content); - } else if (msgType == TSDB_MSG_TYPE_VPEER_CFG) { - mgmtProcessVpeerCfgMsg(content); - } else if (msgType == TSDB_MSG_TYPE_CREATE_RSP) { - mgmtProcessCreateRsp(content); - } else if (msgType == TSDB_MSG_TYPE_REMOVE_RSP) { - // do nothing - } else if (msgType == TSDB_MSG_TYPE_VPEERS_RSP) { - mgmtProcessVPeersRsp(content); - } else if (msgType == TSDB_MSG_TYPE_FREE_VNODE_RSP) { - mgmtProcessFreeVnodeRsp(content); - } else if (msgType == TSDB_MSG_TYPE_CFG_PNODE_RSP) { - // do nothing; - } else if (msgType == TSDB_MSG_TYPE_ALTER_STREAM_RSP) { - // do nothing; - } else { - mError("%s from dnode is not processed", taosMsg[msgType]); - } - - free(sched->msg); -} - -char *mgmtBuildCreateMeterIe(STabObj *pMeter, char *pMsg, int vnode) { - SCreateMsg *pCreateMeter; - - pCreateMeter = (SCreateMsg *)pMsg; - pCreateMeter->vnode = htons(vnode); - pCreateMeter->sid = htonl(pMeter->gid.sid); - pCreateMeter->uid = pMeter->uid; - memcpy(pCreateMeter->meterId, pMeter->meterId, TSDB_METER_ID_LEN); - - // pCreateMeter->lastCreate = htobe64(pVgroup->lastCreate); - pCreateMeter->timeStamp = htobe64(pMeter->createdTime); - /* - pCreateMeter->spi = pSec->spi; - pCreateMeter->encrypt = pSec->encrypt; - memcpy(pCreateMeter->cipheringKey, pSec->cipheringKey, TSDB_KEY_LEN); - memcpy(pCreateMeter->secret, pSec->secret, TSDB_KEY_LEN); - */ - pCreateMeter->sversion = htonl(pMeter->sversion); - pCreateMeter->numOfColumns = htons(pMeter->numOfColumns); - SSchema *pSchema = mgmtGetMeterSchema(pMeter); - - for (int i = 0; i < pMeter->numOfColumns; ++i) { - pCreateMeter->schema[i].type = pSchema[i].type; - /* strcpy(pCreateMeter->schema[i].name, pSchema[i].name); */ - pCreateMeter->schema[i].bytes = htons(pSchema[i].bytes); - pCreateMeter->schema[i].colId = htons(pSchema[i].colId); - } - - pMsg = ((char *)(pCreateMeter->schema)) + pMeter->numOfColumns * sizeof(SMColumn); - pCreateMeter->sqlLen = 0; - - if (pMeter->pSql) { - int len = strlen(pMeter->pSql) + 1; - pCreateMeter->sqlLen = htons(len); - strcpy(pMsg, pMeter->pSql); - pMsg += len; - } - - return pMsg; -} - -int mgmtSendCreateMsgToVnode(STabObj *pMeter, int vnode) { - char *pMsg, *pStart; - - pStart = (char *)malloc(64000); - if (pStart == NULL) return -1; - - *pStart = TSDB_MSG_TYPE_CREATE; - pMsg = pStart + 1; - - pMsg = mgmtBuildCreateMeterIe(pMeter, pMsg, vnode); - mgmtSendMsgToDnode(pStart); - - return 0; -} - -int mgmtSendRemoveMeterMsgToVnode(STabObj *pMeter, int vnode) { - SRemoveMeterMsg *pRemove; - char * pMsg, *pStart; - - pStart = (char *)malloc(1+sizeof(SRemoveMeterMsg)); - if (pStart == NULL) return -1; - - *pStart = TSDB_MSG_TYPE_REMOVE; - pMsg = pStart + 1; - - pRemove = (SRemoveMeterMsg *)pMsg; - pRemove->vnode = htons(vnode); - pRemove->sid = htonl(pMeter->gid.sid); - memcpy(pRemove->meterId, pMeter->meterId, TSDB_METER_ID_LEN); - - mgmtSendMsgToDnode(pStart); - mTrace("vid:%d, send remove meter msg, sid:%d", vnode, pMeter->gid.sid); - - return 0; -} - -int mgmtSendAlterStreamMsgToVnode(STabObj *pMeter, int vnode) { - SAlterStreamMsg *pAlter; - char * pMsg, *pStart; - - pStart = (char *)malloc(128); - if (pStart == NULL) return -1; - - *pStart = TSDB_MSG_TYPE_ALTER_STREAM; - pMsg = pStart + 1; - - pAlter = (SAlterStreamMsg *)pMsg; - pAlter->vnode = htons(vnode); - pAlter->sid = htonl(pMeter->gid.sid); - pAlter->uid = pMeter->uid; - pAlter->status = pMeter->status; - - mgmtSendMsgToDnode(pStart); - - return 0; -} - -char *mgmtBuildVpeersIe(char *pMsg, SVgObj *pVgroup, SDbObj *pDb) { - SVPeersMsg *pVPeers = (SVPeersMsg *)pMsg; - - pVPeers->vnode = htonl(pVgroup->vnodeGid[0].vnode); - - pVPeers->cfg = pDb->cfg; - SVnodeCfg *pCfg = &pVPeers->cfg; - pCfg->vgId = htonl(pVgroup->vgId); - pCfg->maxSessions = htonl(pCfg->maxSessions); - pCfg->cacheBlockSize = htonl(pCfg->cacheBlockSize); - pCfg->cacheNumOfBlocks.totalBlocks = htonl(pCfg->cacheNumOfBlocks.totalBlocks); - pCfg->daysPerFile = htonl(pCfg->daysPerFile); - pCfg->daysToKeep1 = htonl(pCfg->daysToKeep1); - pCfg->daysToKeep2 = htonl(pCfg->daysToKeep2); - pCfg->daysToKeep = htonl(pCfg->daysToKeep); - pCfg->commitTime = htonl(pCfg->commitTime); - pCfg->blocksPerMeter = htons(pCfg->blocksPerMeter); - pCfg->replications = 1; - pCfg->rowsInFileBlock = htonl(pCfg->rowsInFileBlock); - - return pMsg; -} - -int mgmtSendVPeersMsg(SVgObj *pVgroup, SDbObj *pDb) { - char *pMsg, *pStart; - - pStart = (char *)malloc(1024); - if (pStart == NULL) return -1; - - *pStart = TSDB_MSG_TYPE_VPEERS; - pMsg = pStart + 1; - - pMsg = mgmtBuildVpeersIe(pMsg, pVgroup, pDb); - - mgmtSendMsgToDnode(pStart); - - return 0; -} - -int mgmtSendFreeVnodeMsg(int vnode) { - SFreeVnodeMsg *pFreeVnode; - char * pMsg, *pStart; - - pStart = (char *)malloc(128); - if (pStart == NULL) return -1; - - *pStart = TSDB_MSG_TYPE_FREE_VNODE; - pMsg = pStart + 1; - - pFreeVnode = (SFreeVnodeMsg *)pMsg; - pFreeVnode->vnode = htons(vnode); - - mgmtSendMsgToDnode(pStart); - - return 0; -} - -int mgmtCfgDynamicOptions(SDnodeObj *pDnode, char *msg) { return 0; } diff --git a/src/system/src/vnodeFilterFunc.c b/src/system/src/vnodeFilterFunc.c deleted file mode 100644 index 553e194759..0000000000 --- a/src/system/src/vnodeFilterFunc.c +++ /dev/null @@ -1,470 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include -#include - -#include "taosmsg.h" -#include "tsqlfunction.h" -#include "vnode.h" -#include "vnodeDataFilterFunc.h" - -bool less_i8(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int8_t *)minval < pFilter->data.upperBndi); -} - -bool less_i16(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int16_t *)minval < pFilter->data.upperBndi); -} - -bool less_i32(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int32_t *)minval < pFilter->data.upperBndi); -} - -bool less_i64(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int64_t *)minval < pFilter->data.upperBndi); -} - -bool less_ds(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(float *)minval < pFilter->data.upperBndd); -} - -bool less_dd(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(double *)minval < pFilter->data.upperBndd); -} - -////////////////////////////////////////////////////////////////// -bool large_i8(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int8_t *)maxval > pFilter->data.lowerBndi); -} - -bool large_i16(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int16_t *)maxval > pFilter->data.lowerBndi); -} - -bool large_i32(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int32_t *)maxval > pFilter->data.lowerBndi); -} - -bool large_i64(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int64_t *)maxval > pFilter->data.lowerBndi); -} - -bool large_ds(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(float *)maxval > pFilter->data.lowerBndd); -} - -bool large_dd(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(double *)maxval > pFilter->data.lowerBndd); -} -///////////////////////////////////////////////////////////////////// - -bool lessEqual_i8(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int8_t *)minval <= pFilter->data.upperBndi); -} - -bool lessEqual_i16(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int16_t *)minval <= pFilter->data.upperBndi); -} - -bool lessEqual_i32(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int32_t *)minval <= pFilter->data.upperBndi); -} - -bool lessEqual_i64(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int64_t *)minval <= pFilter->data.upperBndi); -} - -bool lessEqual_ds(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(float *)minval <= pFilter->data.upperBndd); -} - -bool lessEqual_dd(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(double *)minval <= pFilter->data.upperBndd); -} - -////////////////////////////////////////////////////////////////////////// -bool largeEqual_i8(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int8_t *)maxval >= pFilter->data.lowerBndi); -} - -bool largeEqual_i16(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int16_t *)maxval >= pFilter->data.lowerBndi); -} - -bool largeEqual_i32(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int32_t *)maxval >= pFilter->data.lowerBndi); -} - -bool largeEqual_i64(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int64_t *)maxval >= pFilter->data.lowerBndi); -} - -bool largeEqual_ds(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(float *)maxval >= pFilter->data.lowerBndd); -} - -bool largeEqual_dd(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(double *)maxval >= pFilter->data.lowerBndd); -} - -//////////////////////////////////////////////////////////////////////// - -bool equal_i8(SColumnFilter *pFilter, char *minval, char *maxval) { - if (*(int8_t *)minval == *(int8_t *)maxval) { - return (*(int8_t *)minval == pFilter->data.lowerBndi); - } else { /* range filter */ - assert(*(int8_t *)minval < *(int8_t *)maxval); - - return *(int8_t *)minval <= pFilter->data.lowerBndi && *(int8_t *)maxval >= pFilter->data.lowerBndi; - } -} - -bool equal_i16(SColumnFilter *pFilter, char *minval, char *maxval) { - if (*(int16_t *)minval == *(int16_t *)maxval) { - return (*(int16_t *)minval == pFilter->data.lowerBndi); - } else { /* range filter */ - assert(*(int16_t *)minval < *(int16_t *)maxval); - - return *(int16_t *)minval <= pFilter->data.lowerBndi && *(int16_t *)maxval >= pFilter->data.lowerBndi; - } -} - -bool equal_i32(SColumnFilter *pFilter, char *minval, char *maxval) { - if (*(int32_t *)minval == *(int32_t *)maxval) { - return (*(int32_t *)minval == pFilter->data.lowerBndi); - } else { /* range filter */ - assert(*(int32_t *)minval < *(int32_t *)maxval); - - return *(int32_t *)minval <= pFilter->data.lowerBndi && *(int32_t *)maxval >= pFilter->data.lowerBndi; - } -} - -bool equal_i64(SColumnFilter *pFilter, char *minval, char *maxval) { - if (*(int64_t *)minval == *(int64_t *)maxval) { - return (*(int64_t *)minval == pFilter->data.lowerBndi); - } else { /* range filter */ - assert(*(int64_t *)minval < *(int64_t *)maxval); - - return *(int64_t *)minval <= pFilter->data.lowerBndi && *(int64_t *)maxval >= pFilter->data.lowerBndi; - } -} - -bool equal_ds(SColumnFilter *pFilter, char *minval, char *maxval) { - if (*(float *)minval == *(float *)maxval) { - return (fabs(*(float *)minval - pFilter->data.lowerBndd) <= FLT_EPSILON); - } else { /* range filter */ - assert(*(float *)minval < *(float *)maxval); - return *(float *)minval <= pFilter->data.lowerBndd && *(float *)maxval >= pFilter->data.lowerBndd; - } -} - -bool equal_dd(SColumnFilter *pFilter, char *minval, char *maxval) { - if (*(double *)minval == *(double *)maxval) { - return (*(double *)minval == pFilter->data.lowerBndd); - } else { /* range filter */ - assert(*(double *)minval < *(double *)maxval); - - return *(double *)minval <= pFilter->data.lowerBndi && *(double *)maxval >= pFilter->data.lowerBndi; - } -} - -bool equal_str(SColumnFilter *pFilter, char *minval, char *maxval) { - // query condition string is greater than the max length of string, not qualified data - if (pFilter->data.len > pFilter->data.bytes) { - return false; - } - - return strncmp((char *)pFilter->data.pz, minval, pFilter->data.bytes) == 0; -} - -//////////////////////////////////////////////////////////////// -bool like_str(SColumnFilter *pFilter, char *minval, char *maxval) { - SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - - return patternMatch((char *)pFilter->data.pz, minval, pFilter->data.bytes, &info) == TSDB_PATTERN_MATCH; -} - -//////////////////////////////////////////////////////////////// -/** - * If minval equals to maxval, it may serve as the one element filter, - * or all elements of an array are identical during pref-filter stage. - * Otherwise, it must be pre-filter of array list of elements. - * - * During pre-filter stage, if there is one element that locates in [minval, maxval], - * the filter function will return true. - */ -bool nequal_i8(SColumnFilter *pFilter, char *minval, char *maxval) { - if (*(int8_t *)minval == *(int8_t *)maxval) { - return (*(int8_t *)minval != pFilter->data.lowerBndi); - } - - return true; -} - -bool nequal_i16(SColumnFilter *pFilter, char *minval, char *maxval) { - if (*(int16_t *)minval == *(int16_t *)maxval) { - return (*(int16_t *)minval != pFilter->data.lowerBndi); - } - - return true; -} - -bool nequal_i32(SColumnFilter *pFilter, char *minval, char *maxval) { - if (*(int32_t *)minval == *(int32_t *)maxval) { - return (*(int32_t *)minval != pFilter->data.lowerBndi); - } - - return true; -} - -bool nequal_i64(SColumnFilter *pFilter, char *minval, char *maxval) { - if (*(int64_t *)minval == *(int64_t *)maxval) { - return (*(int64_t *)minval != pFilter->data.lowerBndi); - } - - return true; -} - -bool nequal_ds(SColumnFilter *pFilter, char *minval, char *maxval) { - if (*(float *)minval == *(float *)maxval) { - return (*(float *)minval != pFilter->data.lowerBndd); - } - - return true; -} - -bool nequal_dd(SColumnFilter *pFilter, char *minval, char *maxval) { - if (*(double *)minval == *(double *)maxval) { - return (*(double *)minval != pFilter->data.lowerBndd); - } - - return true; -} - -bool nequal_str(SColumnFilter *pFilter, char *minval, char *maxval) { - if (pFilter->data.len > pFilter->data.bytes) { - return true; - } - - return strncmp((char *)pFilter->data.pz, minval, pFilter->data.bytes) != 0; -} - -//////////////////////////////////////////////////////////////// - -bool rangeFilter_i32_ii(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int32_t *)minval <= pFilter->data.upperBndi && *(int32_t *)maxval >= pFilter->data.lowerBndi); -} - -bool rangeFilter_i32_ee(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int32_t *)minvaldata.upperBndi &&*(int32_t *)maxval> pFilter->data.lowerBndi); -} - -bool rangeFilter_i32_ie(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int32_t *)minval < pFilter->data.upperBndi && *(int32_t *)maxval >= pFilter->data.lowerBndi); -} - -bool rangeFilter_i32_ei(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int32_t *)minval <= pFilter->data.upperBndi && *(int32_t *)maxval > pFilter->data.lowerBndi); -} - -/////////////////////////////////////////////////////////////////////////////// -bool rangeFilter_i8_ii(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int8_t *)minval <= pFilter->data.upperBndi && *(int8_t *)maxval >= pFilter->data.lowerBndi); -} - -bool rangeFilter_i8_ee(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int8_t *)minvaldata.upperBndi &&*(int8_t *)maxval> pFilter->data.lowerBndi); -} - -bool rangeFilter_i8_ie(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int8_t *)minval < pFilter->data.upperBndi && *(int8_t *)maxval >= pFilter->data.lowerBndi); -} - -bool rangeFilter_i8_ei(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int8_t *)minval <= pFilter->data.upperBndi && *(int8_t *)maxval > pFilter->data.lowerBndi); -} - -///////////////////////////////////////////////////////////////////////////////////// -bool rangeFilter_i16_ii(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int16_t *)minval <= pFilter->data.upperBndi && *(int16_t *)maxval >= pFilter->data.lowerBndi); -} - -bool rangeFilter_i16_ee(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int16_t *)minvaldata.upperBndi &&*(int16_t *)maxval> pFilter->data.lowerBndi); -} - -bool rangeFilter_i16_ie(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int16_t *)minval < pFilter->data.upperBndi && *(int16_t *)maxval >= pFilter->data.lowerBndi); -} - -bool rangeFilter_i16_ei(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int16_t *)minval <= pFilter->data.upperBndi && *(int16_t *)maxval > pFilter->data.lowerBndi); -} - -//////////////////////////////////////////////////////////////////////// -bool rangeFilter_i64_ii(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int64_t *)minval <= pFilter->data.upperBndi && *(int64_t *)maxval >= pFilter->data.lowerBndi); -} - -bool rangeFilter_i64_ee(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int64_t *)minvaldata.upperBndi &&*(int64_t *)maxval> pFilter->data.lowerBndi); -} - -bool rangeFilter_i64_ie(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int64_t *)minval < pFilter->data.upperBndi && *(int64_t *)maxval >= pFilter->data.lowerBndi); -} - -bool rangeFilter_i64_ei(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(int64_t *)minval <= pFilter->data.upperBndi && *(int64_t *)maxval > pFilter->data.lowerBndi); -} - -//////////////////////////////////////////////////////////////////////// -bool rangeFilter_ds_ii(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(float *)minval <= pFilter->data.upperBndd && *(float *)maxval >= pFilter->data.lowerBndd); -} - -bool rangeFilter_ds_ee(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(float *)minvaldata.upperBndd &&*(float *)maxval> pFilter->data.lowerBndd); -} - -bool rangeFilter_ds_ie(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(float *)minval < pFilter->data.upperBndd && *(float *)maxval >= pFilter->data.lowerBndd); -} - -bool rangeFilter_ds_ei(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(float *)minval <= pFilter->data.upperBndd && *(float *)maxval > pFilter->data.lowerBndd); -} - -////////////////////////////////////////////////////////////////////////// -bool rangeFilter_dd_ii(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(double *)minval <= pFilter->data.upperBndd && *(double *)maxval >= pFilter->data.lowerBndd); -} - -bool rangeFilter_dd_ee(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(double *)minvaldata.upperBndd &&*(double *)maxval> pFilter->data.lowerBndd); -} - -bool rangeFilter_dd_ie(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(double *)minval < pFilter->data.upperBndd && *(double *)maxval >= pFilter->data.lowerBndd); -} - -bool rangeFilter_dd_ei(SColumnFilter *pFilter, char *minval, char *maxval) { - return (*(double *)minval <= pFilter->data.upperBndd && *(double *)maxval > pFilter->data.lowerBndd); -} - -//////////////////////////////////////////////////////////////////////////// -bool (*filterFunc_i8[])(SColumnFilter *pFilter, char *minval, char *maxval) = { - NULL, less_i8, large_i8, equal_i8, lessEqual_i8, largeEqual_i8, nequal_i8, NULL, -}; - -bool (*filterFunc_i16[])(SColumnFilter *pFilter, char *minval, char *maxval) = { - NULL, less_i16, large_i16, equal_i16, lessEqual_i16, largeEqual_i16, nequal_i16, NULL, -}; - -bool (*filterFunc_i32[])(SColumnFilter *pFilter, char *minval, char *maxval) = { - NULL, less_i32, large_i32, equal_i32, lessEqual_i32, largeEqual_i32, nequal_i32, NULL, -}; - -bool (*filterFunc_i64[])(SColumnFilter *pFilter, char *minval, char *maxval) = { - NULL, less_i64, large_i64, equal_i64, lessEqual_i64, largeEqual_i64, nequal_i64, NULL, -}; - -bool (*filterFunc_ds[])(SColumnFilter *pFilter, char *minval, char *maxval) = { - NULL, less_ds, large_ds, equal_ds, lessEqual_ds, largeEqual_ds, nequal_ds, NULL, -}; - -bool (*filterFunc_dd[])(SColumnFilter *pFilter, char *minval, char *maxval) = { - NULL, less_dd, large_dd, equal_dd, lessEqual_dd, largeEqual_dd, nequal_dd, NULL, -}; - -bool (*filterFunc_str[])(SColumnFilter *pFilter, char *minval, char *maxval) = { - NULL, NULL, NULL, equal_str, NULL, NULL, nequal_str, like_str, -}; - -bool (*rangeFilterFunc_i8[])(SColumnFilter *pFilter, char *minval, char *maxval) = { - NULL, rangeFilter_i8_ee, rangeFilter_i8_ie, rangeFilter_i8_ei, rangeFilter_i8_ii, -}; - -bool (*rangeFilterFunc_i16[])(SColumnFilter *pFilter, char *minval, char *maxval) = { - NULL, rangeFilter_i16_ee, rangeFilter_i16_ie, rangeFilter_i16_ei, rangeFilter_i16_ii, -}; - -bool (*rangeFilterFunc_i32[])(SColumnFilter *pFilter, char *minval, char *maxval) = { - NULL, rangeFilter_i32_ee, rangeFilter_i32_ie, rangeFilter_i32_ei, rangeFilter_i32_ii, -}; - -bool (*rangeFilterFunc_i64[])(SColumnFilter *pFilter, char *minval, char *maxval) = { - NULL, rangeFilter_i64_ee, rangeFilter_i64_ie, rangeFilter_i64_ei, rangeFilter_i64_ii, -}; - -bool (*rangeFilterFunc_ds[])(SColumnFilter *pFilter, char *minval, char *maxval) = { - NULL, rangeFilter_ds_ee, rangeFilter_ds_ie, rangeFilter_ds_ei, rangeFilter_ds_ii, -}; - -bool (*rangeFilterFunc_dd[])(SColumnFilter *pFilter, char *minval, char *maxval) = { - NULL, rangeFilter_dd_ee, rangeFilter_dd_ie, rangeFilter_dd_ei, rangeFilter_dd_ii, -}; - -__filter_func_t *vnodeGetRangeFilterFuncArray(int32_t type) { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - return rangeFilterFunc_i8; - case TSDB_DATA_TYPE_TINYINT: - return rangeFilterFunc_i8; - case TSDB_DATA_TYPE_SMALLINT: - return rangeFilterFunc_i16; - case TSDB_DATA_TYPE_INT: - return rangeFilterFunc_i32; - case TSDB_DATA_TYPE_TIMESTAMP: // timestamp uses bigint filter - case TSDB_DATA_TYPE_BIGINT: - return rangeFilterFunc_i64; - case TSDB_DATA_TYPE_FLOAT: - return rangeFilterFunc_ds; - case TSDB_DATA_TYPE_DOUBLE: - return rangeFilterFunc_dd; - default: - return NULL; - } -} - -__filter_func_t *vnodeGetValueFilterFuncArray(int32_t type) { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - return filterFunc_i8; - case TSDB_DATA_TYPE_TINYINT: - return filterFunc_i8; - case TSDB_DATA_TYPE_SMALLINT: - return filterFunc_i16; - case TSDB_DATA_TYPE_INT: - return filterFunc_i32; - case TSDB_DATA_TYPE_TIMESTAMP: // timestamp uses bigint filter - case TSDB_DATA_TYPE_BIGINT: - return filterFunc_i64; - case TSDB_DATA_TYPE_FLOAT: - return filterFunc_ds; - case TSDB_DATA_TYPE_DOUBLE: - return filterFunc_dd; - case TSDB_DATA_TYPE_BINARY: - return filterFunc_str; - default: - return NULL; - } -} - -bool vnodeSupportPrefilter(int32_t type) { return type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR; } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt old mode 100755 new mode 100644 index 70e4fd3aa5..7e54759f7e --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -1,43 +1,46 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) - PROJECT(TDengine) -IF (TD_LINUX) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) +INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) + +IF (TD_LINUX_64) AUX_SOURCE_DIRECTORY(src SRC) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/src/inc ${TD_OS_DIR}/inc) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil pthread os m rt) - - FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) - IF (ICONV_INCLUDE_EXIST) + IF (TD_CLUSTER) ADD_DEFINITIONS(-DUSE_LIBICONV) - FIND_PATH(ICONV_LIBRARY_A_EXIST libiconv.a /usr/lib/ /usr/local/lib/ /lib64) - FIND_PATH(ICONV_LIBRARY_SO_EXIST libiconv.so /usr/lib/ /usr/local/lib/ /lib64) - IF (ICONV_LIBRARY_A_EXIST OR ICONV_LIBRARY_SO_EXIST) - MESSAGE(STATUS "Use the installed libiconv library") - TARGET_LINK_LIBRARIES(tutil iconv) + TARGET_LINK_LIBRARIES(tutil iconv) + ELSE() + FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) + IF (ICONV_INCLUDE_EXIST) + ADD_DEFINITIONS(-DUSE_LIBICONV) + FIND_PATH(ICONV_LIBRARY_A_EXIST libiconv.a /usr/lib/ /usr/local/lib/ /lib64) + FIND_PATH(ICONV_LIBRARY_SO_EXIST libiconv.so /usr/lib/ /usr/local/lib/ /lib64) + IF (ICONV_LIBRARY_A_EXIST OR ICONV_LIBRARY_SO_EXIST) + MESSAGE(STATUS "Use the installed libiconv library") + TARGET_LINK_LIBRARIES(tutil iconv) + ELSE () + # libiconv library is already included in GLIBC, + MESSAGE(STATUS "Use the iconv functions in GLIBC") + ENDIF () ELSE () - # libiconv library is already included in GLIBC, - MESSAGE(STATUS "Use the iconv functions in GLIBC") + MESSAGE(STATUS "Failed to find iconv, use default encoding method") ENDIF () - ELSE () - MESSAGE(STATUS "Failed to find iconv, use default encoding method") ENDIF () - -ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows") +ELSEIF (TD_WINDOWS_64) ADD_DEFINITIONS(-DUSE_LIBICONV) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/deps/pthread) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/deps/iconv) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/deps/regex) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/src/inc) - INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/iconv) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/regex) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) LIST(APPEND SRC ./src/ihash.c) LIST(APPEND SRC ./src/lz4.c) LIST(APPEND SRC ./src/shash.c) LIST(APPEND SRC ./src/sql.c) LIST(APPEND SRC ./src/tbase64.c) - LIST(APPEND SRC ./src/tcompression.c) LIST(APPEND SRC ./src/tcache.c) + LIST(APPEND SRC ./src/tcompression.c) LIST(APPEND SRC ./src/textbuffer.c) LIST(APPEND SRC ./src/tglobalcfg.c) LIST(APPEND SRC ./src/thash.c) @@ -48,56 +51,37 @@ ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows") LIST(APPEND SRC ./src/tlog.c) LIST(APPEND SRC ./src/tlosertree.c) LIST(APPEND SRC ./src/tmd5.c) + LIST(APPEND SRC ./src/tmem.c) LIST(APPEND SRC ./src/tmempool.c) LIST(APPEND SRC ./src/tmodule.c) + LIST(APPEND SRC ./src/tnote.c) LIST(APPEND SRC ./src/tsched.c) LIST(APPEND SRC ./src/tskiplist.c) LIST(APPEND SRC ./src/tsocket.c) LIST(APPEND SRC ./src/tstatus.c) LIST(APPEND SRC ./src/tstoken.c) + LIST(APPEND SRC ./src/tstoken.c) + LIST(APPEND SRC ./src/tstrbuild.c) LIST(APPEND SRC ./src/ttime.c) LIST(APPEND SRC ./src/ttimer.c) LIST(APPEND SRC ./src/ttokenizer.c) LIST(APPEND SRC ./src/ttypes.c) LIST(APPEND SRC ./src/tutil.c) LIST(APPEND SRC ./src/version.c) - ADD_LIBRARY(tutil ${SRC}) + ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil iconv regex pthread os winmm IPHLPAPI ws2_32) - -ELSEIF (TD_DARWIN) +ELSEIF(TD_DARWIN_64) ADD_DEFINITIONS(-DUSE_LIBICONV) - INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/src/inc) - INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) - LIST(APPEND SRC ./src/ihash.c) - LIST(APPEND SRC ./src/shash.c) - LIST(APPEND SRC ./src/sql.c) - LIST(APPEND SRC ./src/tbase64.c) - LIST(APPEND SRC ./src/tcache.c) - LIST(APPEND SRC ./src/textbuffer.c) - LIST(APPEND SRC ./src/tglobalcfg.c) - LIST(APPEND SRC ./src/thash.c) - LIST(APPEND SRC ./src/thashutil.c) - LIST(APPEND SRC ./src/thistogram.c) - LIST(APPEND SRC ./src/tidpool.c) - LIST(APPEND SRC ./src/tinterpolation.c) - LIST(APPEND SRC ./src/tlog.c) - LIST(APPEND SRC ./src/tlosertree.c) - LIST(APPEND SRC ./src/tmd5.c) - LIST(APPEND SRC ./src/tmempool.c) - LIST(APPEND SRC ./src/tmodule.c) - LIST(APPEND SRC ./src/tsched.c) - LIST(APPEND SRC ./src/tskiplist.c) - LIST(APPEND SRC ./src/tsocket.c) - LIST(APPEND SRC ./src/tstatus.c) - LIST(APPEND SRC ./src/tstoken.c) - LIST(APPEND SRC ./src/ttime.c) - LIST(APPEND SRC ./src/ttimer.c) - LIST(APPEND SRC ./src/ttokenizer.c) - LIST(APPEND SRC ./src/ttypes.c) - LIST(APPEND SRC ./src/tutil.c) - LIST(APPEND SRC ./src/version.c) + AUX_SOURCE_DIRECTORY(src SRC) + LIST(REMOVE_ITEM SRC ./src/tcrc32c.c) + LIST(REMOVE_ITEM SRC ./src/tdes.c) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil iconv pthread os) +ENDIF() +IF (TD_CLUSTER) + TARGET_LINK_LIBRARIES(tutil tutil_cluster) ENDIF () + + diff --git a/src/util/src/ihash.c b/src/util/src/ihash.c index 200afaa002..606bddd79b 100644 --- a/src/util/src/ihash.c +++ b/src/util/src/ihash.c @@ -39,8 +39,7 @@ typedef struct { int32_t taosHashInt(void *handle, uint64_t key) { IHashObj *pObj = (IHashObj *)handle; - int32_t hash = 0; - hash = key % pObj->maxSessions; + int32_t hash = key % pObj->maxSessions; return hash; } diff --git a/src/util/src/lz4.c b/src/util/src/lz4.c index 6a40456f2f..c48baa63fc 100644 --- a/src/util/src/lz4.c +++ b/src/util/src/lz4.c @@ -32,18 +32,17 @@ - LZ4 source repository : https://github.com/lz4/lz4 */ + /*-************************************ * Tuning parameters **************************************/ /* * LZ4_HEAPMODE : - * Select how default compression functions will allocate memory for their hash - * table, - * in memory stack (0:default, fastest), or in memory heap (1:requires - * malloc()). + * Select how default compression functions will allocate memory for their hash table, + * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). */ #ifndef LZ4_HEAPMODE -#define LZ4_HEAPMODE 0 +# define LZ4_HEAPMODE 0 #endif /* @@ -52,244 +51,218 @@ */ #define ACCELERATION_DEFAULT 1 + /*-************************************ * CPU Feature Detection **************************************/ /* LZ4_FORCE_MEMORY_ACCESS - * By default, access to unaligned memory is controlled by `memcpy()`, which is - * safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly - * is sub-optimal. - * The below switch allow to select different access method for improved - * performance. + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not - * portable). - * This method is safe if your compiler supports it, and *generally* - * as fast or faster than `memcpy`. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets which assembly generation - * depends on alignment. - * But in some circumstances, it's the only known way to get the most - * performance (ie GCC + ARMv6) - * See - * https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html - * for details. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. * Prefer these methods in priority order (0 > 1 > 2) */ -#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ -#if defined(__GNUC__) && \ - (defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || \ - defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || \ - defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__)) -#define LZ4_FORCE_MEMORY_ACCESS 2 -#elif defined(__INTEL_COMPILER) || defined(__GNUC__) -#define LZ4_FORCE_MEMORY_ACCESS 1 -#endif +#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define LZ4_FORCE_MEMORY_ACCESS 2 +# elif defined(__INTEL_COMPILER) || defined(__GNUC__) +# define LZ4_FORCE_MEMORY_ACCESS 1 +# endif #endif /* * LZ4_FORCE_SW_BITCOUNT - * Define this parameter if your target system or compiler does not support - * hardware bit count + * Define this parameter if your target system or compiler does not support hardware bit count */ -#if defined(_MSC_VER) && \ - defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support \ - Hardware bit count */ -#define LZ4_FORCE_SW_BITCOUNT +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ +# define LZ4_FORCE_SW_BITCOUNT #endif + /*-************************************ * Dependency **************************************/ #include "lz4.h" /* see also "memory routines" below */ + /*-************************************ * Compiler Options **************************************/ -#ifdef _MSC_VER /* Visual Studio */ -#include -#pragma warning( \ - disable : 4127) /* disable: C4127: conditional expression is constant */ -#pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) \ - */ -#endif /* _MSC_VER */ +#ifdef _MSC_VER /* Visual Studio */ +# include +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ +#endif /* _MSC_VER */ #ifndef LZ4_FORCE_INLINE -#ifdef _MSC_VER /* Visual Studio */ -#define LZ4_FORCE_INLINE static __forceinline -#else -#if defined(__cplusplus) || \ - defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -#ifdef __GNUC__ -#define LZ4_FORCE_INLINE static inline __attribute__((always_inline)) -#else -#define LZ4_FORCE_INLINE static inline -#endif -#else -#define LZ4_FORCE_INLINE static -#endif /* __STDC_VERSION__ */ -#endif /* _MSC_VER */ +# ifdef _MSC_VER /* Visual Studio */ +# define LZ4_FORCE_INLINE static __forceinline +# else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define LZ4_FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define LZ4_FORCE_INLINE static inline +# endif +# else +# define LZ4_FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +# endif /* _MSC_VER */ #endif /* LZ4_FORCE_INLINE */ -#if (defined(__GNUC__) && (__GNUC__ >= 3)) || \ - (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || \ - defined(__clang__) -#define expect(expr, value) (__builtin_expect((expr), (value))) +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) #else -#define expect(expr, value) (expr) +# define expect(expr,value) (expr) #endif -#define likely(expr) expect((expr) != 0, 1) -#define unlikely(expr) expect((expr) != 0, 0) +#define likely(expr) expect((expr) != 0, 1) +#define unlikely(expr) expect((expr) != 0, 0) + /*-************************************ * Memory routines **************************************/ -#include /* malloc, calloc, free */ -#define ALLOCATOR(n, s) calloc(n, s) -#define FREEMEM free -#include /* memset, memcpy */ -#define MEM_INIT memset +#include /* malloc, calloc, free */ +#define ALLOCATOR(n,s) calloc(n,s) +#define FREEMEM free +#include /* memset, memcpy */ +#define MEM_INIT memset + /*-************************************ * Basic Types **************************************/ -#if defined(__cplusplus) || \ - (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -#include -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -typedef uintptr_t uptrval; +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef uintptr_t uptrval; #else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -typedef size_t uptrval; /* generally true, except OpenVMS-64 */ + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef uint64_t U64; + typedef size_t uptrval; /* generally true, except OpenVMS-64 */ #endif #if defined(__x86_64__) -typedef U64 reg_t; /* 64-bits in x32 mode */ + typedef U64 reg_t; /* 64-bits in x32 mode */ #else -typedef size_t reg_t; /* 32-bits in x32 mode */ + typedef size_t reg_t; /* 32-bits in x32 mode */ #endif /*-************************************ * Reading and writing into memory **************************************/ -static unsigned LZ4_isLittleEndian(void) { - const union { - U32 u; - BYTE c[4]; - } one = {1}; /* don't use static : performance detrimental */ - return one.c[0]; +static unsigned LZ4_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; } -#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS == 2) + +#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) /* lie to the compiler about data alignment; use with caution */ -static U16 LZ4_read16(const void* memPtr) { return *(const U16*)memPtr; } -static U32 LZ4_read32(const void* memPtr) { return *(const U32*)memPtr; } -static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*)memPtr; } +static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } +static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } +static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } -#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS == 1) +#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) -/* __pack instructions are safer, but compiler specific, hence potentially - * problematic for some compilers */ +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ /* currently only defined for gcc and icc */ -typedef union { - U16 u16; - U32 u32; - reg_t uArch; -} __attribute__((packed)) unalign; +typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign; static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -static reg_t LZ4_read_ARCH(const void* ptr) { - return ((const unalign*)ptr)->uArch; -} +static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; } -static void LZ4_write16(void* memPtr, U16 value) { - ((unalign*)memPtr)->u16 = value; -} -static void LZ4_write32(void* memPtr, U32 value) { - ((unalign*)memPtr)->u32 = value; -} +static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } +static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } -#else /* safe and portable access through memcpy() */ +#else /* safe and portable access through memcpy() */ -static U16 LZ4_read16(const void* memPtr) { - U16 val; - memcpy(&val, memPtr, sizeof(val)); - return val; +static U16 LZ4_read16(const void* memPtr) +{ + U16 val; memcpy(&val, memPtr, sizeof(val)); return val; } -static U32 LZ4_read32(const void* memPtr) { - U32 val; - memcpy(&val, memPtr, sizeof(val)); - return val; +static U32 LZ4_read32(const void* memPtr) +{ + U32 val; memcpy(&val, memPtr, sizeof(val)); return val; } -static reg_t LZ4_read_ARCH(const void* memPtr) { - reg_t val; - memcpy(&val, memPtr, sizeof(val)); - return val; +static reg_t LZ4_read_ARCH(const void* memPtr) +{ + reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; } -static void LZ4_write16(void* memPtr, U16 value) { - memcpy(memPtr, &value, sizeof(value)); +static void LZ4_write16(void* memPtr, U16 value) +{ + memcpy(memPtr, &value, sizeof(value)); } -static void LZ4_write32(void* memPtr, U32 value) { - memcpy(memPtr, &value, sizeof(value)); +static void LZ4_write32(void* memPtr, U32 value) +{ + memcpy(memPtr, &value, sizeof(value)); } #endif /* LZ4_FORCE_MEMORY_ACCESS */ -static U16 LZ4_readLE16(const void* memPtr) { - if (LZ4_isLittleEndian()) { - return LZ4_read16(memPtr); - } else { - const BYTE* p = (const BYTE*)memPtr; - return (U16)((U16)p[0] + (p[1] << 8)); - } + +static U16 LZ4_readLE16(const void* memPtr) +{ + if (LZ4_isLittleEndian()) { + return LZ4_read16(memPtr); + } else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)((U16)p[0] + (p[1]<<8)); + } +} + +static void LZ4_writeLE16(void* memPtr, U16 value) +{ + if (LZ4_isLittleEndian()) { + LZ4_write16(memPtr, value); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); + } } -static void LZ4_writeLE16(void* memPtr, U16 value) { - if (LZ4_isLittleEndian()) { - LZ4_write16(memPtr, value); - } else { - BYTE* p = (BYTE*)memPtr; - p[0] = (BYTE)value; - p[1] = (BYTE)(value >> 8); - } +static void LZ4_copy8(void* dst, const void* src) +{ + memcpy(dst,src,8); } -static void LZ4_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } +/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ +static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; -/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd - */ -static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) { - BYTE* d = (BYTE*)dstPtr; - const BYTE* s = (const BYTE*)srcPtr; - BYTE* const e = (BYTE*)dstEnd; - - do { - LZ4_copy8(d, s); - d += 8; - s += 8; - } while (d < e); + do { LZ4_copy8(d,s); d+=8; s+=8; } while (d= 2) -#include -#define DEBUGLOG(l, ...) \ - { \ - if (l <= LZ4_DEBUG) { \ - fprintf(stderr, __FILE__ ": "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, " \n"); \ - } \ - } +#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) +# include +# define DEBUGLOG(l, ...) { \ + if (l<=LZ4_DEBUG) { \ + fprintf(stderr, __FILE__ ": "); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } #else -#define DEBUGLOG(l, ...) \ - {} /* disabled */ +# define DEBUGLOG(l, ...) {} /* disabled */ #endif + /*-************************************ * Common functions **************************************/ -static unsigned LZ4_NbCommonBytes(register reg_t val) { - if (LZ4_isLittleEndian()) { - if (sizeof(val) == 8) { -#if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanForward64(&r, (U64)val); - return (int)(r >> 3); -#elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 3))) && \ - !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); -#else - static const int DeBruijnBytePos[64] = { - 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, - 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, - 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7}; - return DeBruijnBytePos[((U64)((val & -(long long)val) * - 0x0218A392CDABBD3FULL)) >> - 58]; -#endif - } else /* 32 bits */ { -#if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanForward(&r, (U32)val); - return (int)(r >> 3); -#elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 3))) && \ - !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); -#else - static const int DeBruijnBytePos[32] = {0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, - 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, - 2, 0, 3, 1, 2, 0, 1, 0, 1, 1}; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -#endif - } - } else /* Big Endian CPU */ { - if (sizeof(val) == 8) { -#if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse64(&r, val); - return (unsigned)(r >> 3); -#elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 3))) && \ - !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll((U64)val) >> 3); -#else - unsigned r; - if (!(val >> 32)) { - r = 4; - } else { - r = 0; - val >>= 32; - } - if (!(val >> 16)) { - r += 2; - val >>= 8; - } else { - val >>= 24; - } - r += (!val); - return r; -#endif - } else /* 32 bits */ { -#if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse(&r, (unsigned long)val); - return (unsigned)(r >> 3); -#elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 3))) && \ - !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz((U32)val) >> 3); -#else - unsigned r; - if (!(val >> 16)) { - r = 2; - val >>= 8; - } else { - r = 0; - val >>= 24; - } - r += (!val); - return r; -#endif +static unsigned LZ4_NbCommonBytes (register reg_t val) +{ + if (LZ4_isLittleEndian()) { + if (sizeof(val)==8) { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + uint64_t r = 0; + _BitScanForward64( &r, (U64)val ); + return (int)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(int64_t)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + uint64_t r; + _BitScanForward( &r, (U32)val ); + return (int)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } else /* Big Endian CPU */ { + if (sizeof(val)==8) { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + uint64_t r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clzll((U64)val) >> 3); +# else + unsigned r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + uint64_t r = 0; + _BitScanReverse( &r, (uint64_t)val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } } - } } #define STEPSIZE sizeof(reg_t) -static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, - const BYTE* pInLimit) { - const BYTE* const pStart = pIn; - - while (likely(pIn < pInLimit - (STEPSIZE - 1))) { - reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); - if (!diff) { - pIn += STEPSIZE; - pMatch += STEPSIZE; - continue; +static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +{ + const BYTE* const pStart = pIn; + + while (likely(pIn compression run slower on - incompressible data */ +static const int LZ4_64Klimit = ((64 KB) + (MFLIMIT-1)); +static const U32 LZ4_skipTrigger = 6; /* Increase this value ==> compression run slower on incompressible data */ + /*-************************************ * Local Structures and types @@ -472,1092 +404,979 @@ typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; typedef enum { full = 0, partial = 1 } earlyEnd_directive; + /*-************************************ * Local Utils **************************************/ -int LZ4_versionNumber(void) { return LZ4_VERSION_NUMBER; } +int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } -int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } +int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } int LZ4_sizeofState() { return LZ4_STREAMSIZE; } + /*-****************************** * Compression functions ********************************/ -static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) { - if (tableType == byU16) - return ((sequence * 2654435761U) >> ((MINMATCH * 8) - (LZ4_HASHLOG + 1))); - else - return ((sequence * 2654435761U) >> ((MINMATCH * 8) - LZ4_HASHLOG)); +static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) +{ + if (tableType == byU16) + return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); + else + return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); } -static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) { - static const U64 prime5bytes = 889523592379ULL; - static const U64 prime8bytes = 11400714785074694791ULL; - const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG + 1 : LZ4_HASHLOG; - if (LZ4_isLittleEndian()) - return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); - else - return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); +static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) +{ + static const U64 prime5bytes = 889523592379ULL; + static const U64 prime8bytes = 11400714785074694791ULL; + const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; + if (LZ4_isLittleEndian()) + return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); + else + return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); } -LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, - tableType_t const tableType) { - if ((sizeof(reg_t) == 8) && (tableType != byU16)) - return LZ4_hash5(LZ4_read_ARCH(p), tableType); - return LZ4_hash4(LZ4_read32(p), tableType); +LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) +{ + if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); + return LZ4_hash4(LZ4_read32(p), tableType); } -static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, - tableType_t const tableType, - const BYTE* srcBase) { - switch (tableType) { - case byPtr: { - const BYTE** hashTable = (const BYTE**)tableBase; - hashTable[h] = p; - return; - } - case byU32: { - U32* hashTable = (U32*)tableBase; - hashTable[h] = (U32)(p - srcBase); - return; - } - case byU16: { - U16* hashTable = (U16*)tableBase; - hashTable[h] = (U16)(p - srcBase); - return; +static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) +{ + switch (tableType) + { + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } } - } } -LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, - tableType_t tableType, - const BYTE* srcBase) { - U32 const h = LZ4_hashPosition(p, tableType); - LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); +LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 const h = LZ4_hashPosition(p, tableType); + LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); } -static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, - tableType_t tableType, - const BYTE* srcBase) { - if (tableType == byPtr) { - const BYTE** hashTable = (const BYTE**)tableBase; - return hashTable[h]; - } - if (tableType == byU32) { - const U32* const hashTable = (U32*)tableBase; - return hashTable[h] + srcBase; - } - { - const U16* const hashTable = (U16*)tableBase; - return hashTable[h] + srcBase; - } /* default, to ensure a return */ +static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } + if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } + { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ } -LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, - tableType_t tableType, - const BYTE* srcBase) { - U32 const h = LZ4_hashPosition(p, tableType); - return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); +LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 const h = LZ4_hashPosition(p, tableType); + return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); } + /** LZ4_compress_generic() : inlined, to ensure branches are decided at compilation time */ LZ4_FORCE_INLINE int LZ4_compress_generic( - LZ4_stream_t_internal* const cctx, const char* const source, - char* const dest, const int inputSize, const int maxOutputSize, - const limitedOutput_directive outputLimited, const tableType_t tableType, - const dict_directive dict, const dictIssue_directive dictIssue, - const U32 acceleration) { - const BYTE* ip = (const BYTE*)source; - const BYTE* base; - const BYTE* lowLimit; - const BYTE* const lowRefLimit = ip - cctx->dictSize; - const BYTE* const dictionary = cctx->dictionary; - const BYTE* const dictEnd = dictionary + cctx->dictSize; - const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source; - const BYTE* anchor = (const BYTE*)source; - const BYTE* const iend = ip + inputSize; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - - BYTE* op = (BYTE*)dest; - BYTE* const olimit = op + maxOutputSize; - - U32 forwardH; - - /* Init conditions */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) - return 0; /* Unsupported inputSize, too large (or negative) */ - switch (dict) { + LZ4_stream_t_internal* const cctx, + const char* const source, + char* const dest, + const int inputSize, + const int maxOutputSize, + const limitedOutput_directive outputLimited, + const tableType_t tableType, + const dict_directive dict, + const dictIssue_directive dictIssue, + const U32 acceleration) +{ + const BYTE* ip = (const BYTE*) source; + const BYTE* base; + const BYTE* lowLimit; + const BYTE* const lowRefLimit = ip - cctx->dictSize; + const BYTE* const dictionary = cctx->dictionary; + const BYTE* const dictEnd = dictionary + cctx->dictSize; + const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source; + const BYTE* anchor = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + + BYTE* op = (BYTE*) dest; + BYTE* const olimit = op + maxOutputSize; + + U32 forwardH; + + /* Init conditions */ + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ + switch(dict) + { case noDict: default: - base = (const BYTE*)source; - lowLimit = (const BYTE*)source; - break; + base = (const BYTE*)source; + lowLimit = (const BYTE*)source; + break; case withPrefix64k: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source - cctx->dictSize; - break; + base = (const BYTE*)source - cctx->currentOffset; + lowLimit = (const BYTE*)source - cctx->dictSize; + break; case usingExtDict: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source; - break; - } - if ((tableType == byU16) && (inputSize >= LZ4_64Klimit)) - return 0; /* Size too large (not within 64K limit) */ - if (inputSize < LZ4_minLength) - goto _last_literals; /* Input too small, no compression (all literals) */ - - /* First Byte */ - LZ4_putPosition(ip, cctx->hashTable, tableType, base); - ip++; - forwardH = LZ4_hashPosition(ip, tableType); - - /* Main Loop */ - for (;;) { - ptrdiff_t refDelta = 0; - const BYTE* match; - BYTE* token; - - /* Find a match */ - { - const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << LZ4_skipTrigger; - do { - U32 const h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); - if (dict == usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } - } - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); - - } while (((dictIssue == dictSmall) ? (match < lowRefLimit) : 0) || - ((tableType == byU16) ? 0 : (match + MAX_DISTANCE < ip)) || - (LZ4_read32(match + refDelta) != LZ4_read32(ip))); - } - - /* Catch up */ - while (((ip > anchor) & (match + refDelta > lowLimit)) && - (unlikely(ip[-1] == match[refDelta - 1]))) { - ip--; - match--; - } - - /* Encode Literals */ - { - unsigned const litLength = (unsigned)(ip - anchor); - token = op++; - if ((outputLimited) && /* Check output buffer overflow */ - (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + - (litLength / 255) > - olimit))) - return 0; - if (litLength >= RUN_MASK) { - int len = (int)litLength - RUN_MASK; - *token = (RUN_MASK << ML_BITS); - for (; len >= 255; len -= 255) *op++ = 255; - *op++ = (BYTE)len; - } else - *token = (BYTE)(litLength << ML_BITS); - - /* Copy Literals */ - LZ4_wildCopy(op, anchor, op + litLength); - op += litLength; + base = (const BYTE*)source - cctx->currentOffset; + lowLimit = (const BYTE*)source; + break; } + if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (inputSize matchlimit) limit = matchlimit; - matchCode = LZ4_count(ip + MINMATCH, match + MINMATCH, limit); - ip += MINMATCH + matchCode; - if (ip == limit) { - unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); - matchCode += more; - ip += more; + /* First Byte */ + LZ4_putPosition(ip, cctx->hashTable, tableType, base); + ip++; forwardH = LZ4_hashPosition(ip, tableType); + + /* Main Loop */ + for ( ; ; ) { + ptrdiff_t refDelta = 0; + const BYTE* match; + BYTE* token; + + /* Find a match */ + { const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); + if (dict==usingExtDict) { + if (match < (const BYTE*)source) { + refDelta = dictDelta; + lowLimit = dictionary; + } else { + refDelta = 0; + lowLimit = (const BYTE*)source; + } } + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + + } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) + || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); } - } else { - matchCode = LZ4_count(ip + MINMATCH, match + MINMATCH, matchlimit); - ip += MINMATCH + matchCode; - } - if (outputLimited && /* Check output buffer overflow */ - (unlikely(op + (1 + LASTLITERALS) + (matchCode >> 8) > olimit))) - return 0; - if (matchCode >= ML_MASK) { - *token += ML_MASK; - matchCode -= ML_MASK; - LZ4_write32(op, 0xFFFFFFFF); - while (matchCode >= 4 * 255) { - op += 4; - LZ4_write32(op, 0xFFFFFFFF); - matchCode -= 4 * 255; + /* Catch up */ + while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } + + /* Encode Literals */ + { unsigned const litLength = (unsigned)(ip - anchor); + token = op++; + if ((outputLimited) && /* Check output buffer overflow */ + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) + return 0; + if (litLength >= RUN_MASK) { + int len = (int)litLength-RUN_MASK; + *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< mflimit) break; - - /* Fill table */ - LZ4_putPosition(ip - 2, cctx->hashTable, tableType, base); +_next_match: + /* Encode Offset */ + LZ4_writeLE16(op, (U16)(ip-match)); op+=2; + + /* Encode MatchLength */ + { unsigned matchCode; + + if ((dict==usingExtDict) && (lowLimit==dictionary)) { + const BYTE* limit; + match += refDelta; + limit = ip + (dictEnd-match); + if (limit > matchlimit) limit = matchlimit; + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); + ip += MINMATCH + matchCode; + if (ip==limit) { + unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); + matchCode += more; + ip += more; + } + } else { + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + ip += MINMATCH + matchCode; + } + + if ( outputLimited && /* Check output buffer overflow */ + (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) + return 0; + if (matchCode >= ML_MASK) { + *token += ML_MASK; + matchCode -= ML_MASK; + LZ4_write32(op, 0xFFFFFFFF); + while (matchCode >= 4*255) { + op+=4; + LZ4_write32(op, 0xFFFFFFFF); + matchCode -= 4*255; + } + op += matchCode / 255; + *op++ = (BYTE)(matchCode % 255); + } else + *token += (BYTE)(matchCode); + } - /* Test next position */ - match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); - if (dict == usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } + anchor = ip; + + /* Test end of chunk */ + if (ip > mflimit) break; + + /* Fill table */ + LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); + + /* Test next position */ + match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); + if (dict==usingExtDict) { + if (match < (const BYTE*)source) { + refDelta = dictDelta; + lowLimit = dictionary; + } else { + refDelta = 0; + lowLimit = (const BYTE*)source; + } } + LZ4_putPosition(ip, cctx->hashTable, tableType, base); + if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) + && (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); } - LZ4_putPosition(ip, cctx->hashTable, tableType, base); - if (((dictIssue == dictSmall) ? (match >= lowRefLimit) : 1) && - (match + MAX_DISTANCE >= ip) && - (LZ4_read32(match + refDelta) == LZ4_read32(ip))) { - token = op++; - *token = 0; - goto _next_match; - } - - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - } _last_literals: - /* Encode Last Literals */ - { - size_t const lastRun = (size_t)(iend - anchor); - if ((outputLimited) && /* Check output buffer overflow */ - ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun + 255 - RUN_MASK) / 255) > - (U32)maxOutputSize)) - return 0; - if (lastRun >= RUN_MASK) { - size_t accumulator = lastRun - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for (; accumulator >= 255; accumulator -= 255) *op++ = 255; - *op++ = (BYTE)accumulator; - } else { - *op++ = (BYTE)(lastRun << ML_BITS); + /* Encode Last Literals */ + { size_t const lastRun = (size_t)(iend - anchor); + if ( (outputLimited) && /* Check output buffer overflow */ + ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) + return 0; + if (lastRun >= RUN_MASK) { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRun<internal_donotuse; - LZ4_resetStream((LZ4_stream_t*)state); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - if (maxOutputSize >= LZ4_compressBound(inputSize)) { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, - byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, - (sizeof(void*) == 8) ? byU32 : byPtr, noDict, - noDictIssue, acceleration); - } else { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, - limitedOutput, byU16, noDict, noDictIssue, - acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, - limitedOutput, - (sizeof(void*) == 8) ? byU32 : byPtr, noDict, - noDictIssue, acceleration); - } +int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; + LZ4_resetStream((LZ4_stream_t*)state); + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + + if (maxOutputSize >= LZ4_compressBound(inputSize)) { + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } else { + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } } -int LZ4_compress_fast(const char* source, char* dest, int inputSize, - int maxOutputSize, int acceleration) { + +int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ #if (LZ4_HEAPMODE) - void* ctxPtr = ALLOCATOR( - 1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ #else - LZ4_stream_t ctx; - void* const ctxPtr = &ctx; + LZ4_stream_t ctx; + void* const ctxPtr = &ctx; #endif - int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, - maxOutputSize, acceleration); + int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); #if (LZ4_HEAPMODE) - FREEMEM(ctxPtr); + FREEMEM(ctxPtr); #endif - return result; + return result; } -int LZ4_compress_default(const char* source, char* dest, int inputSize, - int maxOutputSize) { - return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); + +int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); } + /* hidden debug function */ -/* strangely enough, gcc generates faster code when this function is - * uncommented, even if unused */ -int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, - int maxOutputSize, int acceleration) { - LZ4_stream_t ctx; - LZ4_resetStream(&ctx); - - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, - maxOutputSize, limitedOutput, byU16, noDict, - noDictIssue, acceleration); - else - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, - maxOutputSize, limitedOutput, - sizeof(void*) == 8 ? byU32 : byPtr, noDict, - noDictIssue, acceleration); +/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ +int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t ctx; + LZ4_resetStream(&ctx); + + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); } + /*-****************************** * *_destSize() variant ********************************/ -static int LZ4_compress_destSize_generic(LZ4_stream_t_internal* const ctx, - const char* const src, char* const dst, - int* const srcSizePtr, - const int targetDstSize, - const tableType_t tableType) { - const BYTE* ip = (const BYTE*)src; - const BYTE* base = (const BYTE*)src; - const BYTE* lowLimit = (const BYTE*)src; - const BYTE* anchor = ip; - const BYTE* const iend = ip + *srcSizePtr; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + targetDstSize; - BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; - BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); - BYTE* const oMaxSeq = oMaxLit - 1 /* token */; - - U32 forwardH; - - /* Init conditions */ - if (targetDstSize < 1) return 0; /* Impossible to store anything */ - if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) - return 0; /* Unsupported input size, too large (or negative) */ - if ((tableType == byU16) && (*srcSizePtr >= LZ4_64Klimit)) - return 0; /* Size too large (not within 64K limit) */ - if (*srcSizePtr < LZ4_minLength) - goto _last_literals; /* Input too small, no compression (all literals) */ - - /* First Byte */ - *srcSizePtr = 0; - LZ4_putPosition(ip, ctx->hashTable, tableType, base); - ip++; - forwardH = LZ4_hashPosition(ip, tableType); - - /* Main Loop */ - for (;;) { - const BYTE* match; - BYTE* token; - - /* Find a match */ - { - const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = 1 << LZ4_skipTrigger; +static int LZ4_compress_destSize_generic( + LZ4_stream_t_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + const int targetDstSize, + const tableType_t tableType) +{ + const BYTE* ip = (const BYTE*) src; + const BYTE* base = (const BYTE*) src; + const BYTE* lowLimit = (const BYTE*) src; + const BYTE* anchor = ip; + const BYTE* const iend = ip + *srcSizePtr; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + targetDstSize; + BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; + BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); + BYTE* const oMaxSeq = oMaxLit - 1 /* token */; + + U32 forwardH; + + + /* Init conditions */ + if (targetDstSize < 1) return 0; /* Impossible to store anything */ + if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ + if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (*srcSizePtrhashTable, tableType, base); + ip++; forwardH = LZ4_hashPosition(ip, tableType); - do { - U32 h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); + /* Main Loop */ + for ( ; ; ) { + const BYTE* match; + BYTE* token; - if (unlikely(forwardIp > mflimit)) goto _last_literals; + /* Find a match */ + { const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = 1 << LZ4_skipTrigger; - match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); + do { + U32 h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); - } while (((tableType == byU16) ? 0 : (match + MAX_DISTANCE < ip)) || - (LZ4_read32(match) != LZ4_read32(ip))); - } + if (unlikely(forwardIp > mflimit)) goto _last_literals; - /* Catch up */ - while ((ip > anchor) && (match > lowLimit) && - (unlikely(ip[-1] == match[-1]))) { - ip--; - match--; - } + match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); - /* Encode Literal length */ - { - unsigned litLength = (unsigned)(ip - anchor); - token = op++; - if (op + ((litLength + 240) / 255) + litLength > oMaxLit) { - /* Not enough space for a last match */ - op--; - goto _last_literals; - } - if (litLength >= RUN_MASK) { - unsigned len = litLength - RUN_MASK; - *token = (RUN_MASK << ML_BITS); - for (; len >= 255; len -= 255) *op++ = 255; - *op++ = (BYTE)len; - } else - *token = (BYTE)(litLength << ML_BITS); - - /* Copy Literals */ - LZ4_wildCopy(op, anchor, op + litLength); - op += litLength; - } + } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + } - _next_match: - /* Encode Offset */ - LZ4_writeLE16(op, (U16)(ip - match)); - op += 2; + /* Catch up */ + while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } + + /* Encode Literal length */ + { unsigned litLength = (unsigned)(ip - anchor); + token = op++; + if (op + ((litLength+240)/255) + litLength > oMaxLit) { + /* Not enough space for a last match */ + op--; + goto _last_literals; + } + if (litLength>=RUN_MASK) { + unsigned len = litLength - RUN_MASK; + *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< oMaxMatch) { - /* Match description too long : reduce it */ - matchLength = (15 - 1) + (oMaxMatch - op) * 255; - } - ip += MINMATCH + matchLength; - - if (matchLength >= ML_MASK) { - *token += ML_MASK; - matchLength -= ML_MASK; - while (matchLength >= 255) { - matchLength -= 255; - *op++ = 255; +_next_match: + /* Encode Offset */ + LZ4_writeLE16(op, (U16)(ip-match)); op+=2; + + /* Encode MatchLength */ + { size_t matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + + if (op + ((matchLength+240)/255) > oMaxMatch) { + /* Match description too long : reduce it */ + matchLength = (15-1) + (oMaxMatch-op) * 255; + } + ip += MINMATCH + matchLength; + + if (matchLength>=ML_MASK) { + *token += ML_MASK; + matchLength -= ML_MASK; + while (matchLength >= 255) { matchLength-=255; *op++ = 255; } + *op++ = (BYTE)matchLength; + } + else *token += (BYTE)(matchLength); } - *op++ = (BYTE)matchLength; - } else - *token += (BYTE)(matchLength); - } - anchor = ip; + anchor = ip; - /* Test end of block */ - if (ip > mflimit) break; - if (op > oMaxSeq) break; + /* Test end of block */ + if (ip > mflimit) break; + if (op > oMaxSeq) break; - /* Fill table */ - LZ4_putPosition(ip - 2, ctx->hashTable, tableType, base); + /* Fill table */ + LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); - /* Test next position */ - match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); - LZ4_putPosition(ip, ctx->hashTable, tableType, base); - if ((match + MAX_DISTANCE >= ip) && (LZ4_read32(match) == LZ4_read32(ip))) { - token = op++; - *token = 0; - goto _next_match; - } + /* Test next position */ + match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); + LZ4_putPosition(ip, ctx->hashTable, tableType, base); + if ( (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match)==LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - } + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + } _last_literals: - /* Encode Last Literals */ - { - size_t lastRunSize = (size_t)(iend - anchor); - if (op + 1 /* token */ + ((lastRunSize + 240) / 255) /* litLength */ + - lastRunSize /* literals */ > oend) { - /* adapt lastRunSize to fill 'dst' */ - lastRunSize = (oend - op) - 1; - lastRunSize -= (lastRunSize + 240) / 255; - } - ip = anchor + lastRunSize; + /* Encode Last Literals */ + { size_t lastRunSize = (size_t)(iend - anchor); + if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) { + /* adapt lastRunSize to fill 'dst' */ + lastRunSize = (oend-op) - 1; + lastRunSize -= (lastRunSize+240)/255; + } + ip = anchor + lastRunSize; - if (lastRunSize >= RUN_MASK) { - size_t accumulator = lastRunSize - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for (; accumulator >= 255; accumulator -= 255) *op++ = 255; - *op++ = (BYTE)accumulator; - } else { - *op++ = (BYTE)(lastRunSize << ML_BITS); + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize<= - LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ - return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, - targetDstSize, 1); - } else { - if (*srcSizePtr < LZ4_64Klimit) - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, - srcSizePtr, targetDstSize, byU16); - else - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, - srcSizePtr, targetDstSize, - sizeof(void*) == 8 ? byU32 : byPtr); - } + +static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize) +{ + LZ4_resetStream(state); + + if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ + return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); + } else { + if (*srcSizePtr < LZ4_64Klimit) + return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); + else + return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr); + } } -int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, - int targetDstSize) { + +int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) +{ #if (LZ4_HEAPMODE) - LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR( - 1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ #else - LZ4_stream_t ctxBody; - LZ4_stream_t* ctx = &ctxBody; + LZ4_stream_t ctxBody; + LZ4_stream_t* ctx = &ctxBody; #endif - int result = - LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); + int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); #if (LZ4_HEAPMODE) - FREEMEM(ctx); + FREEMEM(ctx); #endif - return result; + return result; } + + /*-****************************** * Streaming functions ********************************/ -LZ4_stream_t* LZ4_createStream(void) { - LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); - LZ4_STATIC_ASSERT( - LZ4_STREAMSIZE >= - sizeof(LZ4_stream_t_internal)); /* A compilation error here means - LZ4_STREAMSIZE is not large enough */ - LZ4_resetStream(lz4s); - return lz4s; +LZ4_stream_t* LZ4_createStream(void) +{ + LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); + LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ + LZ4_resetStream(lz4s); + return lz4s; } -void LZ4_resetStream(LZ4_stream_t* LZ4_stream) { - MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); +void LZ4_resetStream (LZ4_stream_t* LZ4_stream) +{ + MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); } -int LZ4_freeStream(LZ4_stream_t* LZ4_stream) { - if (!LZ4_stream) return 0; /* support free on NULL */ - FREEMEM(LZ4_stream); - return (0); +int LZ4_freeStream (LZ4_stream_t* LZ4_stream) +{ + if (!LZ4_stream) return 0; /* support free on NULL */ + FREEMEM(LZ4_stream); + return (0); } + #define HASH_UNIT sizeof(reg_t) -int LZ4_loadDict(LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) { - LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; - const BYTE* p = (const BYTE*)dictionary; - const BYTE* const dictEnd = p + dictSize; - const BYTE* base; - - if ((dict->initCheck) || - (dict->currentOffset > - 1 GB)) /* Uninitialized structure, or reuse overflow */ - LZ4_resetStream(LZ4_dict); - - if (dictSize < (int)HASH_UNIT) { - dict->dictionary = NULL; - dict->dictSize = 0; - return 0; - } +int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) +{ + LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; + const BYTE* p = (const BYTE*)dictionary; + const BYTE* const dictEnd = p + dictSize; + const BYTE* base; + + if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ + LZ4_resetStream(LZ4_dict); + + if (dictSize < (int)HASH_UNIT) { + dict->dictionary = NULL; + dict->dictSize = 0; + return 0; + } - if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; - dict->currentOffset += 64 KB; - base = p - dict->currentOffset; - dict->dictionary = p; - dict->dictSize = (U32)(dictEnd - p); - dict->currentOffset += dict->dictSize; + if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; + dict->currentOffset += 64 KB; + base = p - dict->currentOffset; + dict->dictionary = p; + dict->dictSize = (U32)(dictEnd - p); + dict->currentOffset += dict->dictSize; - while (p <= dictEnd - HASH_UNIT) { - LZ4_putPosition(p, dict->hashTable, byU32, base); - p += 3; - } + while (p <= dictEnd-HASH_UNIT) { + LZ4_putPosition(p, dict->hashTable, byU32, base); + p+=3; + } - return dict->dictSize; + return dict->dictSize; } -static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) { - if ((LZ4_dict->currentOffset > 0x80000000) || - ((uptrval)LZ4_dict->currentOffset > - (uptrval)src)) { /* address space overflow */ - /* rescale hash table */ - U32 const delta = LZ4_dict->currentOffset - 64 KB; - const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; - int i; - for (i = 0; i < LZ4_HASH_SIZE_U32; i++) { - if (LZ4_dict->hashTable[i] < delta) - LZ4_dict->hashTable[i] = 0; - else - LZ4_dict->hashTable[i] -= delta; + +static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) +{ + if ((LZ4_dict->currentOffset > 0x80000000) || + ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */ + /* rescale hash table */ + U32 const delta = LZ4_dict->currentOffset - 64 KB; + const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; + int i; + for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; + else LZ4_dict->hashTable[i] -= delta; + } + LZ4_dict->currentOffset = 64 KB; + if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; + LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; } - LZ4_dict->currentOffset = 64 KB; - if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; - LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; - } } -int LZ4_compress_fast_continue(LZ4_stream_t* LZ4_stream, const char* source, - char* dest, int inputSize, int maxOutputSize, - int acceleration) { - LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - - const BYTE* smallest = (const BYTE*)source; - if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ - if ((streamPtr->dictSize > 0) && (smallest > dictEnd)) smallest = dictEnd; - LZ4_renormDictT(streamPtr, smallest); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - - /* Check overlapping input/dictionary space */ - { - const BYTE* sourceEnd = (const BYTE*)source + inputSize; - if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { - streamPtr->dictSize = (U32)(dictEnd - sourceEnd); - if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; - if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; - streamPtr->dictionary = dictEnd - streamPtr->dictSize; + +int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; + const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + + const BYTE* smallest = (const BYTE*) source; + if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ + if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; + LZ4_renormDictT(streamPtr, smallest); + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + + /* Check overlapping input/dictionary space */ + { const BYTE* sourceEnd = (const BYTE*) source + inputSize; + if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { + streamPtr->dictSize = (U32)(dictEnd - sourceEnd); + if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; + if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; + streamPtr->dictionary = dictEnd - streamPtr->dictSize; + } } - } - /* prefix mode : source data follows dictionary */ - if (dictEnd == (const BYTE*)source) { - int result; - if ((streamPtr->dictSize < 64 KB) && - (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, - maxOutputSize, limitedOutput, byU32, - withPrefix64k, dictSmall, acceleration); - else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, - maxOutputSize, limitedOutput, byU32, - withPrefix64k, noDictIssue, acceleration); - streamPtr->dictSize += (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } + /* prefix mode : source data follows dictionary */ + if (dictEnd == (const BYTE*)source) { + int result; + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); + else + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); + streamPtr->dictSize += (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + return result; + } - /* external dictionary mode */ - { - int result; - if ((streamPtr->dictSize < 64 KB) && - (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, - maxOutputSize, limitedOutput, byU32, - usingExtDict, dictSmall, acceleration); - else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, - maxOutputSize, limitedOutput, byU32, - usingExtDict, noDictIssue, acceleration); - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } + /* external dictionary mode */ + { int result; + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); + else + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + return result; + } } + /* Hidden debug function, to force external dictionary mode */ -int LZ4_compress_forceExtDict(LZ4_stream_t* LZ4_dict, const char* source, - char* dest, int inputSize) { - LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; - int result; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) +{ + LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; + int result; + const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - const BYTE* smallest = dictEnd; - if (smallest > (const BYTE*)source) smallest = (const BYTE*)source; - LZ4_renormDictT(streamPtr, smallest); + const BYTE* smallest = dictEnd; + if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; + LZ4_renormDictT(streamPtr, smallest); - result = - LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, - byU32, usingExtDict, noDictIssue, 1); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; - return result; + return result; } + /*! LZ4_saveDict() : - * If previously compressed data block is not guaranteed to remain available at - * its memory location, + * If previously compressed data block is not guaranteed to remain available at its memory location, * save it into a safer place (char* safeBuffer). * Note : you don't need to call LZ4_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call - * LZ4_compress_fast_continue(). - * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if - * error. + * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). + * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. */ -int LZ4_saveDict(LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) { - LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; - const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; +int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) +{ + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; + const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; - if ((U32)dictSize > 64 KB) - dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ - if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; + if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; - memmove(safeBuffer, previousDictEnd - dictSize, dictSize); + memmove(safeBuffer, previousDictEnd - dictSize, dictSize); - dict->dictionary = (const BYTE*)safeBuffer; - dict->dictSize = (U32)dictSize; + dict->dictionary = (const BYTE*)safeBuffer; + dict->dictSize = (U32)dictSize; - return dictSize; + return dictSize; } + + /*-***************************** * Decompression functions *******************************/ /*! LZ4_decompress_generic() : * This generic decompression function covers all use cases. * It shall be instantiated several times, using different sets of directives. - * Note that it is important for performance that this function really get - * inlined, + * Note that it is important for performance that this function really get inlined, * in order to remove useless branches during compilation optimization. */ LZ4_FORCE_INLINE int LZ4_decompress_generic( - const char* const src, char* const dst, int srcSize, - int outputSize, /* If endOnInput==endOnInputSize, this value is - `dstCapacity` */ - - int endOnInput, /* endOnOutputSize, endOnInputSize */ - int partialDecoding, /* full, partial */ - int targetOutputSize, /* only used if partialDecoding==partial */ - int dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* == dst when no prefix */ - const BYTE* const dictStart, /* only if dict==usingExtDict */ - const size_t dictSize /* note : = 0 if noDict */ - ) { - const BYTE* ip = (const BYTE*)src; - const BYTE* const iend = ip + srcSize; - - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + outputSize; - BYTE* cpy; - BYTE* oexit = op + targetOutputSize; - - const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; - const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3}; - - const int safeDecode = (endOnInput == endOnInputSize); - const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); - - /* Special cases */ - if ((partialDecoding) && (oexit > oend - MFLIMIT)) - oexit = oend - MFLIMIT; /* targetOutputSize too high => decode everything */ - if ((endOnInput) && (unlikely(outputSize == 0))) - return ((srcSize == 1) && (*ip == 0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize == 0))) return (*ip == 0 ? 1 : -1); - - /* Main Loop : decode sequences */ - while (1) { - size_t length; - const BYTE* match; - size_t offset; - - /* get literal length */ - unsigned const token = *ip++; - if ((length = (token >> ML_BITS)) == RUN_MASK) { - unsigned s; - do { - s = *ip++; - length += s; - } while (likely(endOnInput ? ip < iend - RUN_MASK : 1) & (s == 255)); - if ((safeDecode) && unlikely((uptrval)(op) + length < (uptrval)(op))) - goto _output_error; /* overflow detection */ - if ((safeDecode) && unlikely((uptrval)(ip) + length < (uptrval)(ip))) - goto _output_error; /* overflow detection */ - } + const char* const src, + char* const dst, + int srcSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ + + int endOnInput, /* endOnOutputSize, endOnInputSize */ + int partialDecoding, /* full, partial */ + int targetOutputSize, /* only used if partialDecoding==partial */ + int dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* == dst when no prefix */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) +{ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + outputSize; + BYTE* cpy; + BYTE* oexit = op + targetOutputSize; + + const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; + const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; + const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3}; + + const int safeDecode = (endOnInput==endOnInputSize); + const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + + + /* Special cases */ + if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ + if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ + if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); + + /* Main Loop : decode sequences */ + while (1) { + size_t length; + const BYTE* match; + size_t offset; + + /* get literal length */ + unsigned const token = *ip++; + if ((length=(token>>ML_BITS)) == RUN_MASK) { + unsigned s; + do { + s = *ip++; + length += s; + } while ( likely(endOnInput ? ip (partialDecoding ? oexit : oend - MFLIMIT)) || - (ip + length > iend - (2 + 1 + LASTLITERALS)))) || - ((!endOnInput) && (cpy > oend - WILDCOPYLENGTH))) { - if (partialDecoding) { - if (cpy > oend) - goto _output_error; /* Error : write attempt beyond end of output - buffer */ - if ((endOnInput) && (ip + length > iend)) - goto _output_error; /* Error : read attempt beyond end of input buffer - */ - } else { - if ((!endOnInput) && (cpy != oend)) - goto _output_error; /* Error : block decoding must stop exactly there - */ - if ((endOnInput) && ((ip + length != iend) || (cpy > oend))) - goto _output_error; /* Error : input must be consumed */ - } - memcpy(op, ip, length); - ip += length; - op += length; - break; /* Necessarily EOF, due to parsing restrictions */ - } - LZ4_wildCopy(op, ip, cpy); - ip += length; - op = cpy; - - /* get offset */ - offset = LZ4_readLE16(ip); - ip += 2; - match = op - offset; - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) - goto _output_error; /* Error : offset outside buffers */ - LZ4_write32( - op, - (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ - - /* get matchlength */ - length = token & ML_MASK; - if (length == ML_MASK) { - unsigned s; - do { - s = *ip++; - if ((endOnInput) && (ip > iend - LASTLITERALS)) goto _output_error; - length += s; - } while (s == 255); - if ((safeDecode) && unlikely((uptrval)(op) + length < (uptrval)op)) - goto _output_error; /* overflow detection */ - } - length += MINMATCH; - - /* check external dictionary */ - if ((dict == usingExtDict) && (match < lowPrefix)) { - if (unlikely(op + length > oend - LASTLITERALS)) - goto _output_error; /* doesn't respect parsing restriction */ - - if (length <= (size_t)(lowPrefix - match)) { - /* match can be copied as a single segment from external dictionary */ - memmove(op, dictEnd - (lowPrefix - match), length); - op += length; - } else { - /* match encompass external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix - match); - size_t const restSize = length - copySize; - memcpy(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; - } else { - memcpy(op, lowPrefix, restSize); - op += restSize; + /* copy literals */ + cpy = op+length; + if ( ((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) + { + if (partialDecoding) { + if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ + if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + } else { + if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + } + memcpy(op, ip, length); + ip += length; + op += length; + break; /* Necessarily EOF, due to parsing restrictions */ + } + LZ4_wildCopy(op, ip, cpy); + ip += length; op = cpy; + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ + + /* get matchlength */ + length = token & ML_MASK; + if (length == ML_MASK) { + unsigned s; + do { + s = *ip++; + if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; + length += s; + } while (s==255); + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + + /* check external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ + + if (length <= (size_t)(lowPrefix-match)) { + /* match can be copied as a single segment from external dictionary */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match encompass external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix-match); + size_t const restSize = length - copySize; + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; } - } - continue; - } - /* copy match within block */ - cpy = op + length; - if (unlikely(offset < 8)) { - const int dec64 = dec64table[offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[offset]; - memcpy(op + 4, match, 4); - match -= dec64; - } else { - LZ4_copy8(op, match); - match += 8; - } - op += 8; - - if (unlikely(cpy > oend - 12)) { - BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH - 1); - if (cpy > oend - LASTLITERALS) - goto _output_error; /* Error : last LASTLITERALS bytes must be literals - (uncompressed) */ - if (op < oCopyLimit) { - LZ4_wildCopy(op, match, oCopyLimit); - match += oCopyLimit - op; - op = oCopyLimit; - } - while (op < cpy) *op++ = *match++; - } else { - LZ4_copy8(op, match); - if (length > 16) LZ4_wildCopy(op + 8, match + 8, cpy); + /* copy match within block */ + cpy = op + length; + if (unlikely(offset<8)) { + const int dec64 = dec64table[offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[offset]; + memcpy(op+4, match, 4); + match -= dec64; + } else { LZ4_copy8(op, match); match+=8; } + op += 8; + + if (unlikely(cpy>oend-12)) { + BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op16) LZ4_wildCopy(op+8, match+8, cpy); + } + op=cpy; /* correction */ } - op = cpy; /* correction */ - } - /* end of decoding */ - if (endOnInput) - return (int)(((char*)op) - dst); /* Nb of output bytes decoded */ - else - return (int)(((const char*)ip) - src); /* Nb of input bytes read */ + /* end of decoding */ + if (endOnInput) + return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ + else + return (int) (((const char*)ip)-src); /* Nb of input bytes read */ -/* Overflow error detected */ + /* Overflow error detected */ _output_error: - return (int)(-(((const char*)ip) - src)) - 1; + return (int) (-(((const char*)ip)-src))-1; } -int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, - int maxDecompressedSize) { - return LZ4_decompress_generic(source, dest, compressedSize, - maxDecompressedSize, endOnInputSize, full, 0, - noDict, (BYTE*)dest, NULL, 0); + +int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); } -int LZ4_decompress_safe_partial(const char* source, char* dest, - int compressedSize, int targetOutputSize, - int maxDecompressedSize) { - return LZ4_decompress_generic(source, dest, compressedSize, - maxDecompressedSize, endOnInputSize, partial, - targetOutputSize, noDict, (BYTE*)dest, NULL, 0); +int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); } -int LZ4_decompress_fast(const char* source, char* dest, int originalSize) { - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, - full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), - NULL, 64 KB); +int LZ4_decompress_fast(const char* source, char* dest, int originalSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); } + /*===== streaming decompression functions =====*/ -LZ4_streamDecode_t* LZ4_createStreamDecode(void) { - LZ4_streamDecode_t* lz4s = - (LZ4_streamDecode_t*)ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); - return lz4s; +LZ4_streamDecode_t* LZ4_createStreamDecode(void) +{ + LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); + return lz4s; } -int LZ4_freeStreamDecode(LZ4_streamDecode_t* LZ4_stream) { - if (!LZ4_stream) return 0; /* support free on NULL */ - FREEMEM(LZ4_stream); - return 0; +int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) +{ + if (!LZ4_stream) return 0; /* support free on NULL */ + FREEMEM(LZ4_stream); + return 0; } /*! * LZ4_setStreamDecode() : * Use this function to instruct where to find the dictionary. - * This function is not necessary if previous data is still available where it - * was decoded. + * This function is not necessary if previous data is still available where it was decoded. * Loading a size of 0 is allowed (same effect as no dictionary). * Return : 1 if OK, 0 if error */ -int LZ4_setStreamDecode(LZ4_streamDecode_t* LZ4_streamDecode, - const char* dictionary, int dictSize) { - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - lz4sd->prefixSize = (size_t)dictSize; - lz4sd->prefixEnd = (const BYTE*)dictionary + dictSize; - lz4sd->externalDict = NULL; - lz4sd->extDictSize = 0; - return 1; +int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + lz4sd->prefixSize = (size_t) dictSize; + lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; + lz4sd->externalDict = NULL; + lz4sd->extDictSize = 0; + return 1; } /* *_continue() : - These decoding functions allow decompression of multiple blocks in -"streaming" mode. - Previously decoded blocks must still be available at the memory position -where they were decoded. - If it's not possible, save the relevant part of decoded data into a safe -buffer, + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks must still be available at the memory position where they were decoded. + If it's not possible, save the relevant part of decoded data into a safe buffer, and indicate where it stands using LZ4_setStreamDecode() */ -int LZ4_decompress_safe_continue(LZ4_streamDecode_t* LZ4_streamDecode, - const char* source, char* dest, - int compressedSize, int maxOutputSize) { - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - int result; - - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, usingExtDict, - lz4sd->prefixEnd - lz4sd->prefixSize, - lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize += result; - lz4sd->prefixEnd += result; - } else { - lz4sd->extDictSize = lz4sd->prefixSize; - lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic( - source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize = result; - lz4sd->prefixEnd = (BYTE*)dest + result; - } - - return result; +int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + int result; + + if (lz4sd->prefixEnd == (BYTE*)dest) { + result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += result; + lz4sd->prefixEnd += result; + } else { + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } + + return result; } -int LZ4_decompress_fast_continue(LZ4_streamDecode_t* LZ4_streamDecode, - const char* source, char* dest, - int originalSize) { - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - int result; - - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, usingExtDict, - lz4sd->prefixEnd - lz4sd->prefixSize, - lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize += originalSize; - lz4sd->prefixEnd += originalSize; - } else { - lz4sd->extDictSize = lz4sd->prefixSize; - lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic( - source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, - (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize = originalSize; - lz4sd->prefixEnd = (BYTE*)dest + originalSize; - } - - return result; +int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + int result; + + if (lz4sd->prefixEnd == (BYTE*)dest) { + result = LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += originalSize; + lz4sd->prefixEnd += originalSize; + } else { + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } + + return result; } + /* Advanced decoding functions : *_usingDict() : @@ -1565,81 +1384,45 @@ Advanced decoding functions : the dictionary must be explicitly provided within parameters */ -LZ4_FORCE_INLINE int LZ4_decompress_usingDict_generic( - const char* source, char* dest, int compressedSize, int maxOutputSize, - int safe, const char* dictStart, int dictSize) { - if (dictSize == 0) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - safe, full, 0, noDict, (BYTE*)dest, NULL, 0); - if (dictStart + dictSize == dest) { - if (dictSize >= (int)(64 KB - 1)) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - safe, full, 0, withPrefix64k, - (BYTE*)dest - 64 KB, NULL, 0); - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - safe, full, 0, noDict, (BYTE*)dest - dictSize, - NULL, 0); - } - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - safe, full, 0, usingExtDict, (BYTE*)dest, - (const BYTE*)dictStart, dictSize); +LZ4_FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); + if (dictStart+dictSize == dest) { + if (dictSize >= (int)(64 KB - 1)) + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); + } + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } -int LZ4_decompress_safe_usingDict(const char* source, char* dest, - int compressedSize, int maxOutputSize, - const char* dictStart, int dictSize) { - return LZ4_decompress_usingDict_generic( - source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); +int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); } -int LZ4_decompress_fast_usingDict(const char* source, char* dest, - int originalSize, const char* dictStart, - int dictSize) { - return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, - dictStart, dictSize); +int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); } /* debug function */ -int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, - int compressedSize, int maxOutputSize, - const char* dictStart, int dictSize) { - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, usingExtDict, - (BYTE*)dest, (const BYTE*)dictStart, dictSize); +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } + /*=************************************************* * Obsolete Functions ***************************************************/ /* obsolete compression functions */ -int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, - int maxOutputSize) { - return LZ4_compress_default(source, dest, inputSize, maxOutputSize); -} -int LZ4_compress(const char* source, char* dest, int inputSize) { - return LZ4_compress_default(source, dest, inputSize, - LZ4_compressBound(inputSize)); -} -int LZ4_compress_limitedOutput_withState(void* state, const char* src, - char* dst, int srcSize, int dstSize) { - return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); -} -int LZ4_compress_withState(void* state, const char* src, char* dst, - int srcSize) { - return LZ4_compress_fast_extState(state, src, dst, srcSize, - LZ4_compressBound(srcSize), 1); -} -int LZ4_compress_limitedOutput_continue(LZ4_stream_t* LZ4_stream, - const char* src, char* dst, int srcSize, - int maxDstSize) { - return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, - 1); -} -int LZ4_compress_continue(LZ4_stream_t* LZ4_stream, const char* source, - char* dest, int inputSize) { - return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, - LZ4_compressBound(inputSize), 1); -} +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } +int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); } +int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } +int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); } +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } /* These function names are deprecated and should no longer be used. @@ -1647,57 +1430,51 @@ They are only provided here for compatibility with older user programs. - LZ4_uncompress is totally equivalent to LZ4_decompress_fast - LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe */ -int LZ4_uncompress(const char* source, char* dest, int outputSize) { - return LZ4_decompress_fast(source, dest, outputSize); -} -int LZ4_uncompress_unknownOutputSize(const char* source, char* dest, int isize, - int maxOutputSize) { - return LZ4_decompress_safe(source, dest, isize, maxOutputSize); -} +int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } +int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } + /* Obsolete Streaming functions */ int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } -static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) { - MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); - lz4ds->internal_donotuse.bufferStart = base; +static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) +{ + MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); + lz4ds->internal_donotuse.bufferStart = base; } -int LZ4_resetStreamState(void* state, char* inputBuffer) { - if ((((uptrval)state) & 3) != 0) - return 1; /* Error : pointer is not aligned on 4-bytes boundary */ - LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); - return 0; +int LZ4_resetStreamState(void* state, char* inputBuffer) +{ + if ((((uptrval)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ + LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); + return 0; } -void* LZ4_create(char* inputBuffer) { - LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); - LZ4_init(lz4ds, (BYTE*)inputBuffer); - return lz4ds; +void* LZ4_create (char* inputBuffer) +{ + LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); + LZ4_init (lz4ds, (BYTE*)inputBuffer); + return lz4ds; } -char* LZ4_slideInputBuffer(void* LZ4_Data) { - LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; - int dictSize = - LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); - return (char*)(ctx->bufferStart + dictSize); +char* LZ4_slideInputBuffer (void* LZ4_Data) +{ + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; + int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); + return (char*)(ctx->bufferStart + dictSize); } /* Obsolete streaming decompression functions */ -int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, - int compressedSize, int maxOutputSize) { - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, withPrefix64k, - (BYTE*)dest - 64 KB, NULL, 64 KB); +int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); } -int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, - int originalSize) { - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, - full, 0, withPrefix64k, (BYTE*)dest - 64 KB, - NULL, 64 KB); +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); } -#endif /* LZ4_COMMONDEFS_ONLY */ +#endif /* LZ4_COMMONDEFS_ONLY */ diff --git a/src/util/src/sql.c b/src/util/src/sql.c index 8cf76d44ab..7e90e5f939 100644 --- a/src/util/src/sql.c +++ b/src/util/src/sql.c @@ -91,26 +91,27 @@ # define INTERFACE 1 #endif /************* Begin control #defines *****************************************/ -#define YYCODETYPE unsigned char -#define YYNOCODE 241 +#define YYCODETYPE unsigned short int +#define YYNOCODE 261 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SSQLToken typedef union { int yyinit; ParseTOKENTYPE yy0; SQuerySQL* yy24; - tSQLExprList* yy98; - tFieldList* yy151; - int64_t yy189; - tVariantList* yy216; - tVariant yy266; - SCreateTableSQL* yy278; + tVariantList* yy56; + tSQLExprListList* yy74; + tSQLExpr* yy90; + SCreateTableSQL* yy158; + tVariant yy186; + TAOS_FIELD yy223; + SCreateAcctSQL yy279; SLimitVal yy294; - TAOS_FIELD yy343; - tSQLExpr* yy370; - int yy412; - tSQLExprListList* yy434; - SCreateDBInfo yy478; + int yy332; + int64_t yy389; + SCreateDBInfo yy398; + tFieldList* yy471; + tSQLExprList* yy498; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -120,16 +121,16 @@ typedef union { #define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo #define ParseARG_STORE yypParser->pInfo = pInfo #define YYFALLBACK 1 -#define YYNSTATE 218 -#define YYNRULE 180 -#define YY_MAX_SHIFT 217 -#define YY_MIN_SHIFTREDUCE 348 -#define YY_MAX_SHIFTREDUCE 527 -#define YY_MIN_REDUCE 528 -#define YY_MAX_REDUCE 707 -#define YY_ERROR_ACTION 708 -#define YY_ACCEPT_ACTION 709 -#define YY_NO_ACTION 710 +#define YYNSTATE 251 +#define YYNRULE 213 +#define YY_MAX_SHIFT 250 +#define YY_MIN_SHIFTREDUCE 401 +#define YY_MAX_SHIFTREDUCE 613 +#define YY_MIN_REDUCE 614 +#define YY_MAX_REDUCE 826 +#define YY_ERROR_ACTION 827 +#define YY_ACCEPT_ACTION 828 +#define YY_NO_ACTION 829 /************* End control #defines *******************************************/ /* The yyzerominor constant is used to initialize instances of @@ -201,176 +202,196 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (474) +#define YY_ACTTAB_COUNT (529) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 380, 31, 30, 709, 217, 29, 28, 27, 381, 68, - /* 10 */ 69, 75, 38, 40, 380, 32, 33, 212, 438, 70, - /* 20 */ 26, 54, 381, 183, 36, 34, 37, 35, 133, 216, - /* 30 */ 436, 7, 31, 30, 58, 99, 29, 28, 27, 38, - /* 40 */ 40, 432, 32, 33, 429, 10, 430, 26, 431, 115, - /* 50 */ 183, 36, 34, 37, 35, 443, 89, 139, 510, 31, - /* 60 */ 30, 113, 115, 29, 28, 27, 38, 40, 149, 32, - /* 70 */ 33, 509, 134, 151, 26, 115, 45, 183, 36, 34, - /* 80 */ 37, 35, 89, 138, 510, 112, 31, 30, 380, 53, - /* 90 */ 29, 28, 27, 46, 440, 40, 381, 32, 33, 465, - /* 100 */ 89, 178, 26, 114, 136, 183, 36, 34, 37, 35, - /* 110 */ 102, 103, 61, 478, 31, 30, 428, 146, 29, 28, - /* 120 */ 27, 20, 20, 180, 150, 55, 89, 205, 204, 428, - /* 130 */ 119, 433, 349, 350, 351, 352, 353, 354, 355, 356, - /* 140 */ 357, 358, 359, 464, 147, 148, 425, 425, 210, 32, - /* 150 */ 33, 29, 28, 27, 26, 199, 157, 183, 36, 34, - /* 160 */ 37, 35, 484, 165, 487, 162, 31, 30, 426, 56, - /* 170 */ 29, 28, 27, 17, 198, 197, 196, 195, 194, 193, - /* 180 */ 192, 191, 190, 189, 412, 506, 401, 402, 403, 404, - /* 190 */ 405, 406, 407, 408, 409, 410, 411, 143, 491, 113, - /* 200 */ 200, 482, 11, 485, 48, 488, 20, 143, 491, 416, - /* 210 */ 135, 482, 428, 485, 57, 488, 459, 460, 132, 49, - /* 220 */ 505, 20, 77, 76, 126, 23, 427, 20, 504, 140, - /* 230 */ 141, 424, 131, 182, 109, 107, 78, 208, 207, 140, - /* 240 */ 141, 143, 491, 441, 206, 482, 425, 485, 129, 488, - /* 250 */ 211, 22, 425, 514, 215, 214, 368, 24, 515, 450, - /* 260 */ 480, 451, 24, 42, 508, 15, 492, 142, 14, 171, - /* 270 */ 14, 130, 167, 140, 141, 36, 34, 37, 35, 128, - /* 280 */ 120, 422, 39, 31, 30, 42, 421, 29, 28, 27, - /* 290 */ 21, 490, 39, 483, 187, 486, 481, 2, 21, 67, - /* 300 */ 66, 490, 9, 8, 121, 434, 489, 435, 74, 73, - /* 310 */ 122, 123, 124, 125, 117, 524, 489, 111, 118, 116, - /* 320 */ 413, 475, 442, 474, 86, 98, 39, 144, 471, 470, - /* 330 */ 145, 457, 419, 209, 456, 490, 100, 101, 387, 188, - /* 340 */ 110, 203, 523, 64, 522, 520, 104, 168, 80, 105, - /* 350 */ 489, 378, 377, 71, 375, 374, 446, 152, 106, 372, - /* 360 */ 371, 170, 50, 47, 437, 370, 363, 172, 108, 176, - /* 370 */ 367, 365, 41, 181, 84, 445, 179, 458, 177, 175, - /* 380 */ 173, 25, 22, 202, 185, 90, 44, 213, 527, 153, - /* 390 */ 154, 51, 155, 156, 127, 59, 62, 526, 379, 373, - /* 400 */ 158, 79, 81, 369, 159, 423, 160, 161, 1, 96, - /* 410 */ 93, 91, 92, 94, 525, 95, 97, 163, 164, 518, - /* 420 */ 166, 12, 13, 169, 85, 447, 87, 137, 174, 4, - /* 430 */ 18, 452, 88, 5, 493, 3, 19, 16, 184, 6, - /* 440 */ 186, 60, 399, 398, 397, 396, 395, 394, 393, 392, - /* 450 */ 391, 42, 390, 389, 384, 21, 418, 201, 417, 63, - /* 460 */ 415, 52, 382, 72, 361, 43, 528, 530, 530, 530, - /* 470 */ 530, 82, 65, 83, + /* 0 */ 439, 36, 35, 153, 249, 34, 33, 32, 440, 34, + /* 10 */ 33, 32, 43, 45, 49, 37, 38, 74, 78, 244, + /* 20 */ 31, 85, 77, 205, 41, 39, 42, 40, 80, 133, + /* 30 */ 101, 50, 36, 35, 527, 171, 34, 33, 32, 43, + /* 40 */ 45, 154, 37, 38, 114, 115, 224, 31, 65, 68, + /* 50 */ 205, 41, 39, 42, 40, 76, 133, 828, 250, 36, + /* 60 */ 35, 241, 241, 34, 33, 32, 43, 45, 155, 37, + /* 70 */ 38, 128, 126, 245, 31, 89, 88, 205, 41, 39, + /* 80 */ 42, 40, 202, 524, 59, 135, 36, 35, 439, 21, + /* 90 */ 34, 33, 32, 520, 159, 596, 440, 10, 57, 172, + /* 100 */ 135, 135, 227, 226, 101, 45, 439, 37, 38, 158, + /* 110 */ 596, 595, 31, 156, 440, 205, 41, 39, 42, 40, + /* 120 */ 232, 167, 564, 507, 36, 35, 166, 21, 34, 33, + /* 130 */ 32, 510, 402, 403, 404, 405, 406, 407, 408, 409, + /* 140 */ 410, 411, 412, 413, 510, 37, 38, 243, 132, 508, + /* 150 */ 31, 220, 101, 205, 41, 39, 42, 40, 551, 168, + /* 160 */ 200, 507, 36, 35, 97, 134, 34, 33, 32, 510, + /* 170 */ 21, 139, 101, 17, 219, 242, 218, 217, 216, 215, + /* 180 */ 214, 213, 212, 211, 492, 21, 481, 482, 483, 484, + /* 190 */ 485, 486, 487, 488, 489, 490, 491, 163, 577, 11, + /* 200 */ 243, 568, 228, 571, 507, 574, 550, 163, 577, 498, + /* 210 */ 21, 568, 509, 571, 193, 574, 148, 233, 7, 507, + /* 220 */ 561, 62, 111, 87, 86, 142, 60, 178, 242, 160, + /* 230 */ 161, 147, 437, 204, 186, 124, 183, 230, 229, 160, + /* 240 */ 161, 163, 577, 525, 506, 568, 570, 571, 573, 574, + /* 250 */ 41, 39, 42, 40, 495, 61, 494, 27, 36, 35, + /* 260 */ 545, 546, 34, 33, 32, 514, 28, 600, 511, 162, + /* 270 */ 512, 29, 513, 160, 161, 192, 446, 566, 188, 124, + /* 280 */ 248, 247, 422, 438, 522, 150, 124, 18, 601, 536, + /* 290 */ 537, 44, 29, 47, 15, 594, 169, 170, 578, 14, + /* 300 */ 576, 44, 14, 569, 518, 572, 519, 2, 52, 516, + /* 310 */ 576, 517, 504, 567, 503, 575, 47, 592, 22, 591, + /* 320 */ 209, 73, 72, 53, 22, 575, 9, 8, 84, 83, + /* 330 */ 590, 151, 152, 140, 501, 44, 610, 141, 560, 143, + /* 340 */ 144, 145, 146, 137, 576, 131, 138, 136, 164, 557, + /* 350 */ 556, 165, 526, 231, 110, 98, 112, 113, 448, 575, + /* 360 */ 543, 542, 210, 129, 515, 25, 223, 225, 609, 70, + /* 370 */ 189, 608, 606, 116, 466, 26, 23, 130, 435, 79, + /* 380 */ 433, 81, 431, 191, 430, 173, 125, 428, 427, 426, + /* 390 */ 424, 91, 532, 194, 198, 54, 417, 127, 51, 521, + /* 400 */ 421, 203, 419, 46, 102, 95, 201, 530, 103, 531, + /* 410 */ 544, 195, 199, 197, 30, 27, 222, 235, 75, 234, + /* 420 */ 236, 207, 238, 55, 237, 239, 240, 246, 149, 613, + /* 430 */ 63, 66, 174, 429, 175, 176, 90, 92, 177, 423, + /* 440 */ 119, 612, 118, 467, 117, 120, 121, 179, 122, 123, + /* 450 */ 1, 505, 108, 104, 105, 106, 107, 109, 24, 180, + /* 460 */ 181, 182, 611, 184, 185, 604, 12, 13, 187, 190, + /* 470 */ 96, 533, 99, 157, 58, 538, 196, 100, 19, 4, + /* 480 */ 579, 3, 16, 20, 64, 5, 206, 6, 208, 479, + /* 490 */ 478, 477, 476, 475, 474, 473, 472, 470, 47, 221, + /* 500 */ 443, 67, 445, 22, 500, 48, 499, 497, 464, 56, + /* 510 */ 462, 454, 69, 460, 456, 71, 458, 452, 450, 471, + /* 520 */ 469, 82, 441, 425, 415, 93, 614, 616, 94, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 1, 33, 34, 187, 188, 37, 38, 39, 9, 61, - /* 10 */ 62, 63, 13, 14, 1, 16, 17, 69, 91, 71, - /* 20 */ 21, 94, 9, 24, 25, 26, 27, 28, 189, 190, - /* 30 */ 214, 86, 33, 34, 89, 90, 37, 38, 39, 13, - /* 40 */ 14, 2, 16, 17, 5, 228, 7, 21, 9, 228, - /* 50 */ 24, 25, 26, 27, 28, 190, 190, 236, 237, 33, - /* 60 */ 34, 228, 228, 37, 38, 39, 13, 14, 60, 16, - /* 70 */ 17, 237, 239, 34, 21, 228, 91, 24, 25, 26, - /* 80 */ 27, 28, 190, 236, 237, 228, 33, 34, 1, 90, - /* 90 */ 37, 38, 39, 108, 229, 14, 9, 16, 17, 233, - /* 100 */ 190, 235, 21, 228, 197, 24, 25, 26, 27, 28, - /* 110 */ 61, 62, 63, 87, 33, 34, 209, 197, 37, 38, - /* 120 */ 39, 190, 190, 231, 116, 233, 190, 119, 120, 209, - /* 130 */ 228, 92, 45, 46, 47, 48, 49, 50, 51, 52, - /* 140 */ 53, 54, 55, 233, 213, 213, 215, 215, 190, 16, - /* 150 */ 17, 37, 38, 39, 21, 57, 115, 24, 25, 26, - /* 160 */ 27, 28, 5, 122, 7, 124, 33, 34, 210, 233, - /* 170 */ 37, 38, 39, 75, 76, 77, 78, 79, 80, 81, - /* 180 */ 82, 83, 84, 85, 196, 228, 198, 199, 200, 201, - /* 190 */ 202, 203, 204, 205, 206, 207, 208, 1, 2, 228, - /* 200 */ 197, 5, 44, 7, 91, 9, 190, 1, 2, 5, - /* 210 */ 239, 5, 209, 7, 216, 9, 101, 102, 60, 106, - /* 220 */ 228, 190, 64, 65, 66, 227, 209, 190, 228, 33, - /* 230 */ 34, 215, 74, 37, 61, 62, 63, 33, 34, 33, - /* 240 */ 34, 1, 2, 37, 213, 5, 215, 7, 228, 9, - /* 250 */ 213, 93, 215, 87, 57, 58, 59, 91, 87, 87, - /* 260 */ 1, 87, 91, 91, 87, 91, 87, 56, 91, 111, - /* 270 */ 91, 228, 114, 33, 34, 25, 26, 27, 28, 121, - /* 280 */ 228, 87, 86, 33, 34, 91, 87, 37, 38, 39, - /* 290 */ 91, 95, 86, 5, 87, 7, 37, 86, 91, 117, - /* 300 */ 118, 95, 117, 118, 228, 5, 110, 7, 67, 68, - /* 310 */ 228, 228, 228, 228, 228, 209, 110, 228, 228, 228, - /* 320 */ 209, 211, 190, 211, 190, 217, 86, 211, 211, 211, - /* 330 */ 211, 234, 212, 211, 234, 95, 190, 190, 190, 190, - /* 340 */ 190, 190, 190, 190, 190, 190, 190, 113, 56, 190, - /* 350 */ 110, 190, 190, 190, 190, 190, 95, 190, 190, 190, - /* 360 */ 190, 238, 105, 107, 226, 190, 190, 230, 190, 230, - /* 370 */ 190, 190, 104, 99, 191, 191, 103, 191, 98, 97, - /* 380 */ 96, 109, 93, 72, 191, 225, 77, 72, 5, 123, - /* 390 */ 5, 191, 123, 70, 191, 194, 194, 5, 193, 191, - /* 400 */ 123, 192, 192, 191, 5, 214, 123, 70, 195, 219, - /* 410 */ 222, 224, 223, 221, 5, 220, 218, 123, 70, 76, - /* 420 */ 115, 86, 86, 113, 112, 87, 86, 1, 86, 100, - /* 430 */ 91, 87, 86, 100, 87, 86, 91, 86, 88, 86, - /* 440 */ 88, 67, 9, 5, 5, 5, 5, 1, 5, 5, - /* 450 */ 5, 91, 5, 5, 73, 91, 5, 15, 5, 118, - /* 460 */ 87, 86, 73, 70, 56, 16, 0, 240, 240, 240, - /* 470 */ 240, 21, 118, 21, + /* 0 */ 1, 33, 34, 198, 199, 37, 38, 39, 9, 37, + /* 10 */ 38, 39, 13, 14, 100, 16, 17, 62, 63, 64, + /* 20 */ 21, 66, 67, 24, 25, 26, 27, 28, 73, 247, + /* 30 */ 199, 117, 33, 34, 199, 61, 37, 38, 39, 13, + /* 40 */ 14, 259, 16, 17, 62, 63, 64, 21, 66, 67, + /* 50 */ 24, 25, 26, 27, 28, 71, 247, 196, 197, 33, + /* 60 */ 34, 77, 77, 37, 38, 39, 13, 14, 259, 16, + /* 70 */ 17, 62, 63, 64, 21, 66, 67, 24, 25, 26, + /* 80 */ 27, 28, 251, 248, 253, 247, 33, 34, 1, 199, + /* 90 */ 37, 38, 39, 232, 256, 257, 9, 247, 99, 125, + /* 100 */ 247, 247, 128, 129, 199, 14, 1, 16, 17, 256, + /* 110 */ 257, 257, 21, 216, 9, 24, 25, 26, 27, 28, + /* 120 */ 199, 231, 96, 233, 33, 34, 216, 199, 37, 38, + /* 130 */ 39, 234, 45, 46, 47, 48, 49, 50, 51, 52, + /* 140 */ 53, 54, 55, 56, 234, 16, 17, 58, 247, 228, + /* 150 */ 21, 216, 199, 24, 25, 26, 27, 28, 253, 231, + /* 160 */ 255, 233, 33, 34, 199, 247, 37, 38, 39, 234, + /* 170 */ 199, 247, 199, 84, 85, 86, 87, 88, 89, 90, + /* 180 */ 91, 92, 93, 94, 215, 199, 217, 218, 219, 220, + /* 190 */ 221, 222, 223, 224, 225, 226, 227, 1, 2, 44, + /* 200 */ 58, 5, 231, 7, 233, 9, 253, 1, 2, 5, + /* 210 */ 199, 5, 234, 7, 249, 9, 61, 231, 95, 233, + /* 220 */ 229, 98, 99, 68, 69, 70, 253, 124, 86, 33, + /* 230 */ 34, 76, 203, 37, 131, 206, 133, 33, 34, 33, + /* 240 */ 34, 1, 2, 37, 233, 5, 5, 7, 7, 9, + /* 250 */ 25, 26, 27, 28, 217, 235, 219, 102, 33, 34, + /* 260 */ 110, 111, 37, 38, 39, 2, 246, 96, 5, 57, + /* 270 */ 7, 100, 9, 33, 34, 120, 203, 1, 123, 206, + /* 280 */ 58, 59, 60, 203, 100, 130, 206, 103, 96, 96, + /* 290 */ 96, 95, 100, 100, 100, 96, 33, 34, 96, 100, + /* 300 */ 104, 95, 100, 5, 5, 7, 7, 95, 100, 5, + /* 310 */ 104, 7, 96, 37, 96, 119, 100, 247, 100, 247, + /* 320 */ 96, 126, 127, 115, 100, 119, 126, 127, 71, 72, + /* 330 */ 247, 247, 247, 247, 230, 95, 234, 247, 229, 247, + /* 340 */ 247, 247, 247, 247, 104, 247, 247, 247, 229, 229, + /* 350 */ 229, 229, 199, 229, 236, 199, 199, 199, 199, 119, + /* 360 */ 254, 254, 199, 199, 101, 199, 199, 199, 199, 199, + /* 370 */ 122, 199, 199, 199, 199, 199, 199, 199, 199, 199, + /* 380 */ 199, 199, 199, 258, 199, 199, 199, 199, 199, 199, + /* 390 */ 199, 57, 104, 250, 250, 114, 199, 199, 116, 245, + /* 400 */ 199, 108, 199, 113, 244, 200, 112, 200, 243, 200, + /* 410 */ 200, 105, 107, 106, 118, 102, 74, 49, 83, 82, + /* 420 */ 79, 200, 53, 200, 81, 80, 78, 74, 200, 5, + /* 430 */ 204, 204, 132, 200, 5, 132, 201, 201, 65, 200, + /* 440 */ 208, 5, 212, 214, 213, 211, 209, 132, 210, 207, + /* 450 */ 205, 232, 238, 242, 241, 240, 239, 237, 202, 5, + /* 460 */ 132, 65, 5, 132, 65, 85, 95, 95, 124, 122, + /* 470 */ 121, 96, 95, 1, 100, 96, 95, 95, 100, 109, + /* 480 */ 96, 95, 95, 100, 71, 109, 97, 95, 97, 9, + /* 490 */ 5, 5, 5, 5, 1, 5, 5, 5, 100, 15, + /* 500 */ 75, 71, 65, 100, 5, 16, 5, 96, 5, 95, + /* 510 */ 5, 5, 127, 5, 5, 127, 5, 5, 5, 5, + /* 520 */ 5, 65, 75, 65, 57, 21, 0, 260, 21, }; -#define YY_SHIFT_USE_DFLT (-74) -#define YY_SHIFT_COUNT (217) -#define YY_SHIFT_MIN (-73) -#define YY_SHIFT_MAX (466) +#define YY_SHIFT_USE_DFLT (-87) +#define YY_SHIFT_COUNT (250) +#define YY_SHIFT_MIN (-86) +#define YY_SHIFT_MAX (526) static const short yy_shift_ofst[] = { - /* 0 */ 158, 98, 196, 240, 13, 13, 13, 13, 13, 13, - /* 10 */ -1, 87, 240, 240, 240, 39, 39, 39, 13, 13, - /* 20 */ 13, 13, -74, 206, 240, 240, 240, 240, 240, 240, + /* 0 */ 155, 89, 196, 240, 105, 105, 105, 105, 105, 105, + /* 10 */ -1, 87, 240, 240, 240, 263, 263, 263, 105, 105, + /* 20 */ 105, 105, 105, -16, 142, -15, -15, -87, 206, 240, /* 30 */ 240, 240, 240, 240, 240, 240, 240, 240, 240, 240, - /* 40 */ 240, 240, 39, 39, 39, 204, 204, 204, 204, 204, - /* 50 */ 204, -55, 204, 13, 13, 115, 115, -73, 13, 13, - /* 60 */ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, - /* 70 */ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, - /* 80 */ 13, 13, 13, 13, 234, 292, 292, 261, 261, 292, - /* 90 */ 257, 256, 268, 274, 273, 280, 282, 284, 272, 289, - /* 100 */ 292, 292, 311, 311, 292, 309, 292, 315, 292, 315, - /* 110 */ -74, 26, 53, 53, 53, 53, 53, 81, 133, 250, - /* 120 */ 250, 250, -32, -32, -32, -32, -52, 8, 41, 114, - /* 130 */ 114, 49, 173, 197, 166, 171, 172, 174, 177, 179, - /* 140 */ 157, 288, 259, 211, -15, 113, 194, 199, 207, 182, - /* 150 */ 185, 300, 241, 383, 266, 385, 269, 323, 392, 277, - /* 160 */ 399, 283, 337, 409, 294, 348, 343, 305, 335, 336, - /* 170 */ 310, 312, 338, 340, 426, 342, 344, 346, 339, 329, - /* 180 */ 345, 333, 347, 349, 351, 350, 353, 352, 374, 433, - /* 190 */ 438, 439, 440, 441, 446, 443, 444, 445, 447, 448, - /* 200 */ 360, 381, 442, 449, 341, 354, 364, 451, 453, 373, - /* 210 */ 375, 364, 393, 389, 450, 452, 408, 466, + /* 40 */ 240, 240, 240, 240, 240, 240, 240, 263, 263, 204, + /* 50 */ 204, 204, 204, 204, 204, 123, 204, 105, 105, 150, + /* 60 */ 150, 184, 105, 105, 105, 105, 105, 105, 105, 105, + /* 70 */ 105, 105, 105, 105, 105, 105, 105, 105, 105, 105, + /* 80 */ 105, 105, 105, 105, 105, 105, 105, 105, 105, 105, + /* 90 */ 105, 105, 105, 105, 105, 248, 334, 334, 334, 288, + /* 100 */ 288, 334, 281, 282, 290, 293, 294, 305, 307, 306, + /* 110 */ 296, 313, 334, 334, 342, 342, 334, 335, 337, 368, + /* 120 */ 341, 343, 369, 345, 348, 334, 353, 334, 353, -87, + /* 130 */ -87, 26, 53, 53, 53, 53, 53, 91, 129, 225, + /* 140 */ 225, 225, -45, -32, -32, -32, -32, -18, 9, -26, + /* 150 */ 103, -28, -28, 222, 171, 192, 193, 194, 199, 202, + /* 160 */ 241, 298, 276, 212, -86, 208, 216, 218, 224, 299, + /* 170 */ 304, 195, 200, 257, 424, 300, 429, 303, 373, 436, + /* 180 */ 315, 454, 328, 396, 457, 331, 399, 380, 344, 371, + /* 190 */ 372, 347, 349, 374, 375, 377, 472, 381, 379, 382, + /* 200 */ 378, 370, 383, 376, 384, 386, 387, 389, 392, 391, + /* 210 */ 413, 480, 485, 486, 487, 488, 493, 490, 491, 492, + /* 220 */ 398, 425, 484, 430, 437, 489, 385, 388, 403, 499, + /* 230 */ 501, 411, 414, 403, 503, 505, 506, 508, 509, 511, + /* 240 */ 512, 513, 514, 515, 456, 458, 447, 504, 507, 467, + /* 250 */ 526, }; -#define YY_REDUCE_USE_DFLT (-185) -#define YY_REDUCE_COUNT (110) -#define YY_REDUCE_MIN (-184) -#define YY_REDUCE_MAX (213) +#define YY_REDUCE_USE_DFLT (-219) +#define YY_REDUCE_COUNT (130) +#define YY_REDUCE_MIN (-218) +#define YY_REDUCE_MAX (256) static const short yy_reduce_ofst[] = { - /* 0 */ -184, -12, -179, -153, -134, -108, -69, -68, 31, 37, - /* 10 */ -135, -161, -167, -29, -166, -93, -80, 3, -90, -64, - /* 20 */ -42, 16, -2, -183, -143, -125, -98, -43, -8, 0, - /* 30 */ 20, 43, 52, 76, 82, 83, 84, 85, 86, 89, - /* 40 */ 90, 91, 17, 106, 111, 110, 112, 116, 117, 118, - /* 50 */ 119, 120, 122, 132, 134, 97, 100, 108, 146, 147, - /* 60 */ 148, 149, 150, 151, 152, 153, 154, 155, 156, 159, - /* 70 */ 161, 162, 163, 164, 165, 167, 168, 169, 170, 175, - /* 80 */ 176, 178, 180, 181, 123, 183, 184, 137, 139, 186, - /* 90 */ 138, 160, 187, 189, 188, 192, 195, 190, 198, 191, - /* 100 */ 193, 200, 201, 202, 203, 205, 208, 209, 212, 210, - /* 110 */ 213, + /* 0 */ -139, -31, -162, -147, -95, -169, -110, -72, -29, -14, + /* 10 */ -165, -195, -218, -191, -146, -103, -90, -65, -35, -47, + /* 20 */ -27, -79, 11, 29, 37, 73, 80, 20, -150, -99, + /* 30 */ -82, -76, 70, 72, 83, 84, 85, 86, 90, 92, + /* 40 */ 93, 94, 95, 96, 98, 99, 100, -22, 102, -9, + /* 50 */ 109, 119, 120, 121, 122, 104, 124, 153, 156, 106, + /* 60 */ 107, 118, 157, 158, 159, 163, 164, 166, 167, 168, + /* 70 */ 169, 170, 172, 173, 174, 175, 176, 177, 178, 179, + /* 80 */ 180, 181, 182, 183, 185, 186, 187, 188, 189, 190, + /* 90 */ 191, 197, 198, 201, 203, 125, 205, 207, 209, 143, + /* 100 */ 144, 210, 154, 160, 165, 211, 213, 215, 217, 214, + /* 110 */ 220, 219, 221, 223, 226, 227, 228, 229, 231, 230, + /* 120 */ 232, 234, 237, 238, 242, 233, 235, 239, 236, 245, + /* 130 */ 256, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 708, 566, 692, 692, 708, 708, 708, 708, 708, 708, - /* 10 */ 624, 540, 708, 708, 692, 708, 708, 708, 708, 708, - /* 20 */ 708, 708, 619, 708, 708, 708, 708, 708, 708, 708, - /* 30 */ 708, 708, 708, 708, 708, 708, 708, 708, 708, 708, - /* 40 */ 708, 708, 708, 708, 708, 708, 708, 708, 708, 708, - /* 50 */ 708, 708, 708, 708, 708, 641, 641, 708, 708, 708, - /* 60 */ 708, 708, 708, 708, 708, 708, 708, 708, 708, 708, - /* 70 */ 708, 556, 708, 708, 708, 708, 708, 708, 708, 708, - /* 80 */ 708, 708, 708, 708, 708, 542, 542, 708, 708, 542, - /* 90 */ 648, 652, 646, 634, 642, 633, 629, 628, 656, 708, - /* 100 */ 542, 542, 565, 565, 542, 708, 542, 563, 542, 563, - /* 110 */ 580, 708, 696, 697, 657, 691, 647, 675, 674, 687, - /* 120 */ 681, 680, 679, 678, 677, 676, 708, 708, 708, 683, - /* 130 */ 682, 708, 708, 708, 708, 708, 708, 708, 708, 708, - /* 140 */ 708, 708, 708, 659, 653, 649, 708, 708, 708, 708, - /* 150 */ 708, 708, 708, 708, 708, 708, 708, 708, 708, 708, - /* 160 */ 708, 708, 708, 708, 708, 708, 708, 708, 708, 708, - /* 170 */ 693, 708, 708, 708, 708, 708, 708, 708, 643, 708, - /* 180 */ 635, 708, 708, 708, 708, 708, 708, 600, 708, 708, - /* 190 */ 708, 708, 708, 708, 708, 708, 708, 708, 708, 708, - /* 200 */ 568, 708, 708, 708, 708, 708, 701, 708, 708, 708, - /* 210 */ 594, 699, 708, 708, 546, 544, 708, 708, + /* 0 */ 827, 660, 811, 811, 827, 827, 827, 827, 827, 827, + /* 10 */ 741, 627, 827, 827, 811, 827, 827, 827, 827, 827, + /* 20 */ 827, 827, 827, 662, 649, 662, 662, 736, 827, 827, + /* 30 */ 827, 827, 827, 827, 827, 827, 827, 827, 827, 827, + /* 40 */ 827, 827, 827, 827, 827, 827, 827, 827, 827, 827, + /* 50 */ 827, 827, 827, 827, 827, 827, 827, 827, 827, 760, + /* 60 */ 760, 827, 827, 827, 827, 827, 827, 827, 827, 827, + /* 70 */ 827, 827, 827, 827, 827, 827, 827, 827, 827, 647, + /* 80 */ 827, 645, 827, 827, 827, 827, 827, 827, 827, 827, + /* 90 */ 827, 827, 827, 827, 827, 827, 629, 629, 629, 827, + /* 100 */ 827, 629, 767, 771, 765, 753, 761, 752, 748, 747, + /* 110 */ 775, 827, 629, 629, 657, 657, 629, 678, 676, 674, + /* 120 */ 666, 672, 668, 670, 664, 629, 655, 629, 655, 693, + /* 130 */ 706, 827, 815, 816, 776, 810, 766, 794, 793, 806, + /* 140 */ 800, 799, 827, 798, 797, 796, 795, 827, 827, 827, + /* 150 */ 827, 802, 801, 827, 827, 827, 827, 827, 827, 827, + /* 160 */ 827, 827, 827, 778, 772, 768, 827, 827, 827, 827, + /* 170 */ 827, 827, 827, 827, 827, 827, 827, 827, 827, 827, + /* 180 */ 827, 827, 827, 827, 827, 827, 827, 827, 827, 827, + /* 190 */ 827, 812, 827, 742, 827, 827, 827, 827, 827, 827, + /* 200 */ 762, 827, 754, 827, 827, 827, 827, 827, 827, 715, + /* 210 */ 827, 827, 827, 827, 827, 827, 827, 827, 827, 827, + /* 220 */ 681, 827, 827, 827, 827, 827, 827, 827, 820, 827, + /* 230 */ 827, 827, 709, 818, 827, 827, 827, 827, 827, 827, + /* 240 */ 827, 827, 827, 827, 827, 827, 827, 633, 631, 827, + /* 250 */ 827, }; /********** End of lemon-generated parsing tables *****************************/ @@ -438,6 +459,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* DATABASES => nothing */ 0, /* MNODES => nothing */ 0, /* DNODES => nothing */ + 0, /* ACCOUNTS => nothing */ 0, /* USERS => nothing */ 0, /* MODULES => nothing */ 0, /* QUERIES => nothing */ @@ -453,18 +475,26 @@ static const YYCODETYPE yyFallback[] = { 0, /* DROP => nothing */ 0, /* TABLE => nothing */ 1, /* DATABASE => ID */ + 0, /* DNODE => nothing */ + 1, /* IP => ID */ 0, /* USER => nothing */ + 0, /* ACCOUNT => nothing */ 0, /* USE => nothing */ 0, /* DESCRIBE => nothing */ 0, /* ALTER => nothing */ 0, /* PASS => nothing */ 0, /* PRIVILEGE => nothing */ - 0, /* DNODE => nothing */ - 1, /* IP => ID */ 0, /* LOCAL => nothing */ 0, /* IF => nothing */ 0, /* EXISTS => nothing */ 0, /* CREATE => nothing */ + 0, /* PPS => nothing */ + 0, /* TSERIES => nothing */ + 0, /* DBS => nothing */ + 0, /* STORAGE => nothing */ + 0, /* QTIME => nothing */ + 0, /* CONNS => nothing */ + 0, /* STATE => nothing */ 0, /* KEEP => nothing */ 0, /* CACHE => nothing */ 0, /* REPLICA => nothing */ @@ -563,7 +593,7 @@ static const YYCODETYPE yyFallback[] = { 1, /* HISTOGRAM => ID */ 1, /* DIFF => ID */ 1, /* SPREAD => ID */ - 1, /* WAVG => ID */ + 1, /* TWA => ID */ 1, /* INTERP => ID */ 1, /* LAST_ROW => ID */ 1, /* SEMI => ID */ @@ -670,52 +700,57 @@ static const char *const yyTokenName[] = { "TIMES", "STAR", "SLASH", "REM", "CONCAT", "UMINUS", "UPLUS", "BITNOT", "SHOW", "DATABASES", "MNODES", "DNODES", - "USERS", "MODULES", "QUERIES", "CONNECTIONS", - "STREAMS", "CONFIGS", "SCORES", "GRANTS", - "DOT", "TABLES", "STABLES", "VGROUPS", - "DROP", "TABLE", "DATABASE", "USER", + "ACCOUNTS", "USERS", "MODULES", "QUERIES", + "CONNECTIONS", "STREAMS", "CONFIGS", "SCORES", + "GRANTS", "DOT", "TABLES", "STABLES", + "VGROUPS", "DROP", "TABLE", "DATABASE", + "DNODE", "IP", "USER", "ACCOUNT", "USE", "DESCRIBE", "ALTER", "PASS", - "PRIVILEGE", "DNODE", "IP", "LOCAL", - "IF", "EXISTS", "CREATE", "KEEP", - "CACHE", "REPLICA", "DAYS", "ROWS", - "ABLOCKS", "TBLOCKS", "CTIME", "CLOG", - "COMP", "PRECISION", "LP", "RP", - "TAGS", "USING", "AS", "COMMA", - "NULL", "SELECT", "FROM", "VARIABLE", - "INTERVAL", "FILL", "SLIDING", "ORDER", - "BY", "ASC", "DESC", "GROUP", - "HAVING", "LIMIT", "OFFSET", "SLIMIT", - "SOFFSET", "WHERE", "NOW", "INSERT", - "INTO", "VALUES", "RESET", "QUERY", - "ADD", "COLUMN", "TAG", "CHANGE", - "SET", "KILL", "CONNECTION", "COLON", - "STREAM", "ABORT", "AFTER", "ATTACH", - "BEFORE", "BEGIN", "CASCADE", "CLUSTER", - "CONFLICT", "COPY", "DEFERRED", "DELIMITERS", - "DETACH", "EACH", "END", "EXPLAIN", - "FAIL", "FOR", "IGNORE", "IMMEDIATE", - "INITIALLY", "INSTEAD", "MATCH", "KEY", - "OF", "RAISE", "REPLACE", "RESTRICT", - "ROW", "STATEMENT", "TRIGGER", "VIEW", - "ALL", "COUNT", "SUM", "AVG", - "MIN", "MAX", "FIRST", "LAST", - "TOP", "BOTTOM", "STDDEV", "PERCENTILE", - "APERCENTILE", "LEASTSQUARES", "HISTOGRAM", "DIFF", - "SPREAD", "WAVG", "INTERP", "LAST_ROW", - "SEMI", "NONE", "PREV", "LINEAR", - "IMPORT", "METRIC", "TBNAME", "JOIN", - "METRICS", "STABLE", "error", "program", - "cmd", "dbPrefix", "ids", "cpxName", - "ifexists", "alter_db_optr", "ifnotexists", "db_optr", - "keep", "tagitemlist", "tables", "cache", - "replica", "days", "rows", "ablocks", - "tblocks", "ctime", "clog", "comp", - "prec", "tagitem", "typename", "signed", - "create_table_args", "columnlist", "select", "column", - "selcollist", "from", "where_opt", "interval_opt", - "fill_opt", "sliding_opt", "groupby_opt", "orderby_opt", - "having_opt", "slimit_opt", "limit_opt", "sclp", - "expr", "as", "tmvar", "sortlist", + "PRIVILEGE", "LOCAL", "IF", "EXISTS", + "CREATE", "PPS", "TSERIES", "DBS", + "STORAGE", "QTIME", "CONNS", "STATE", + "KEEP", "CACHE", "REPLICA", "DAYS", + "ROWS", "ABLOCKS", "TBLOCKS", "CTIME", + "CLOG", "COMP", "PRECISION", "LP", + "RP", "TAGS", "USING", "AS", + "COMMA", "NULL", "SELECT", "FROM", + "VARIABLE", "INTERVAL", "FILL", "SLIDING", + "ORDER", "BY", "ASC", "DESC", + "GROUP", "HAVING", "LIMIT", "OFFSET", + "SLIMIT", "SOFFSET", "WHERE", "NOW", + "INSERT", "INTO", "VALUES", "RESET", + "QUERY", "ADD", "COLUMN", "TAG", + "CHANGE", "SET", "KILL", "CONNECTION", + "COLON", "STREAM", "ABORT", "AFTER", + "ATTACH", "BEFORE", "BEGIN", "CASCADE", + "CLUSTER", "CONFLICT", "COPY", "DEFERRED", + "DELIMITERS", "DETACH", "EACH", "END", + "EXPLAIN", "FAIL", "FOR", "IGNORE", + "IMMEDIATE", "INITIALLY", "INSTEAD", "MATCH", + "KEY", "OF", "RAISE", "REPLACE", + "RESTRICT", "ROW", "STATEMENT", "TRIGGER", + "VIEW", "ALL", "COUNT", "SUM", + "AVG", "MIN", "MAX", "FIRST", + "LAST", "TOP", "BOTTOM", "STDDEV", + "PERCENTILE", "APERCENTILE", "LEASTSQUARES", "HISTOGRAM", + "DIFF", "SPREAD", "TWA", "INTERP", + "LAST_ROW", "SEMI", "NONE", "PREV", + "LINEAR", "IMPORT", "METRIC", "TBNAME", + "JOIN", "METRICS", "STABLE", "error", + "program", "cmd", "dbPrefix", "ids", + "cpxName", "ifexists", "alter_db_optr", "acct_optr", + "ifnotexists", "db_optr", "pps", "tseries", + "dbs", "streams", "storage", "qtime", + "users", "conns", "state", "keep", + "tagitemlist", "tables", "cache", "replica", + "days", "rows", "ablocks", "tblocks", + "ctime", "clog", "comp", "prec", + "typename", "signed", "create_table_args", "columnlist", + "select", "column", "tagitem", "selcollist", + "from", "where_opt", "interval_opt", "fill_opt", + "sliding_opt", "groupby_opt", "orderby_opt", "having_opt", + "slimit_opt", "limit_opt", "sclp", "expr", + "as", "tablelist", "tmvar", "sortlist", "sortitem", "item", "sortorder", "grouplist", "exprlist", "expritem", "insert_value_list", "itemlist", }; @@ -729,182 +764,215 @@ static const char *const yyRuleName[] = { /* 1 */ "cmd ::= SHOW DATABASES", /* 2 */ "cmd ::= SHOW MNODES", /* 3 */ "cmd ::= SHOW DNODES", - /* 4 */ "cmd ::= SHOW USERS", - /* 5 */ "cmd ::= SHOW MODULES", - /* 6 */ "cmd ::= SHOW QUERIES", - /* 7 */ "cmd ::= SHOW CONNECTIONS", - /* 8 */ "cmd ::= SHOW STREAMS", - /* 9 */ "cmd ::= SHOW CONFIGS", - /* 10 */ "cmd ::= SHOW SCORES", - /* 11 */ "cmd ::= SHOW GRANTS", - /* 12 */ "dbPrefix ::=", - /* 13 */ "dbPrefix ::= ids DOT", - /* 14 */ "cpxName ::=", - /* 15 */ "cpxName ::= DOT ids", - /* 16 */ "cmd ::= SHOW dbPrefix TABLES", - /* 17 */ "cmd ::= SHOW dbPrefix TABLES LIKE ids", - /* 18 */ "cmd ::= SHOW dbPrefix STABLES", - /* 19 */ "cmd ::= SHOW dbPrefix STABLES LIKE ids", - /* 20 */ "cmd ::= SHOW dbPrefix VGROUPS", - /* 21 */ "cmd ::= DROP TABLE ifexists ids cpxName", - /* 22 */ "cmd ::= DROP DATABASE ifexists ids", - /* 23 */ "cmd ::= DROP USER ids", - /* 24 */ "cmd ::= USE ids", - /* 25 */ "cmd ::= DESCRIBE ids cpxName", - /* 26 */ "cmd ::= ALTER USER ids PASS ids", - /* 27 */ "cmd ::= ALTER USER ids PRIVILEGE ids", - /* 28 */ "cmd ::= ALTER DNODE IP ids", - /* 29 */ "cmd ::= ALTER DNODE IP ids ids", - /* 30 */ "cmd ::= ALTER LOCAL ids", - /* 31 */ "cmd ::= ALTER DATABASE ids alter_db_optr", - /* 32 */ "ids ::= ID", - /* 33 */ "ids ::= STRING", - /* 34 */ "ifexists ::= IF EXISTS", - /* 35 */ "ifexists ::=", - /* 36 */ "ifnotexists ::= IF NOT EXISTS", - /* 37 */ "ifnotexists ::=", - /* 38 */ "cmd ::= CREATE DATABASE ifnotexists ids db_optr", - /* 39 */ "cmd ::= CREATE USER ids PASS ids", - /* 40 */ "keep ::= KEEP tagitemlist", - /* 41 */ "tables ::= TABLES INTEGER", - /* 42 */ "cache ::= CACHE INTEGER", - /* 43 */ "replica ::= REPLICA INTEGER", - /* 44 */ "days ::= DAYS INTEGER", - /* 45 */ "rows ::= ROWS INTEGER", - /* 46 */ "ablocks ::= ABLOCKS ID", - /* 47 */ "tblocks ::= TBLOCKS INTEGER", - /* 48 */ "ctime ::= CTIME INTEGER", - /* 49 */ "clog ::= CLOG INTEGER", - /* 50 */ "comp ::= COMP INTEGER", - /* 51 */ "prec ::= PRECISION STRING", - /* 52 */ "db_optr ::=", - /* 53 */ "db_optr ::= db_optr tables", - /* 54 */ "db_optr ::= db_optr cache", - /* 55 */ "db_optr ::= db_optr replica", - /* 56 */ "db_optr ::= db_optr days", - /* 57 */ "db_optr ::= db_optr rows", - /* 58 */ "db_optr ::= db_optr ablocks", - /* 59 */ "db_optr ::= db_optr tblocks", - /* 60 */ "db_optr ::= db_optr ctime", - /* 61 */ "db_optr ::= db_optr clog", - /* 62 */ "db_optr ::= db_optr comp", - /* 63 */ "db_optr ::= db_optr prec", - /* 64 */ "db_optr ::= db_optr keep", - /* 65 */ "alter_db_optr ::= REPLICA tagitem", - /* 66 */ "typename ::= ids", - /* 67 */ "typename ::= ids LP signed RP", - /* 68 */ "signed ::= INTEGER", - /* 69 */ "signed ::= PLUS INTEGER", - /* 70 */ "signed ::= MINUS INTEGER", - /* 71 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args", - /* 72 */ "create_table_args ::= LP columnlist RP", - /* 73 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP", - /* 74 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP", - /* 75 */ "create_table_args ::= AS select", - /* 76 */ "columnlist ::= columnlist COMMA column", - /* 77 */ "columnlist ::= column", - /* 78 */ "column ::= ids typename", - /* 79 */ "tagitemlist ::= tagitemlist COMMA tagitem", - /* 80 */ "tagitemlist ::= tagitem", - /* 81 */ "tagitem ::= INTEGER", - /* 82 */ "tagitem ::= FLOAT", - /* 83 */ "tagitem ::= STRING", - /* 84 */ "tagitem ::= BOOL", - /* 85 */ "tagitem ::= NULL", - /* 86 */ "tagitem ::= MINUS INTEGER", - /* 87 */ "tagitem ::= MINUS FLOAT", - /* 88 */ "cmd ::= select", - /* 89 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", - /* 90 */ "sclp ::= selcollist COMMA", - /* 91 */ "sclp ::=", - /* 92 */ "selcollist ::= sclp expr as", - /* 93 */ "selcollist ::= sclp STAR", - /* 94 */ "as ::= AS ids", - /* 95 */ "as ::= ids", - /* 96 */ "as ::=", - /* 97 */ "from ::= FROM ids cpxName", - /* 98 */ "tmvar ::= VARIABLE", - /* 99 */ "interval_opt ::= INTERVAL LP tmvar RP", - /* 100 */ "interval_opt ::=", - /* 101 */ "fill_opt ::=", - /* 102 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 103 */ "fill_opt ::= FILL LP ID RP", - /* 104 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 105 */ "sliding_opt ::=", - /* 106 */ "orderby_opt ::=", - /* 107 */ "orderby_opt ::= ORDER BY sortlist", - /* 108 */ "sortlist ::= sortlist COMMA item sortorder", - /* 109 */ "sortlist ::= item sortorder", - /* 110 */ "item ::= ids cpxName", - /* 111 */ "sortorder ::= ASC", - /* 112 */ "sortorder ::= DESC", - /* 113 */ "sortorder ::=", - /* 114 */ "groupby_opt ::=", - /* 115 */ "groupby_opt ::= GROUP BY grouplist", - /* 116 */ "grouplist ::= grouplist COMMA item", - /* 117 */ "grouplist ::= item", - /* 118 */ "having_opt ::=", - /* 119 */ "having_opt ::= HAVING expr", - /* 120 */ "limit_opt ::=", - /* 121 */ "limit_opt ::= LIMIT signed", - /* 122 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 123 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 124 */ "slimit_opt ::=", - /* 125 */ "slimit_opt ::= SLIMIT signed", - /* 126 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 127 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 128 */ "where_opt ::=", - /* 129 */ "where_opt ::= WHERE expr", - /* 130 */ "expr ::= LP expr RP", - /* 131 */ "expr ::= ID", - /* 132 */ "expr ::= ID DOT ID", - /* 133 */ "expr ::= ID DOT STAR", - /* 134 */ "expr ::= INTEGER", - /* 135 */ "expr ::= MINUS INTEGER", - /* 136 */ "expr ::= PLUS INTEGER", - /* 137 */ "expr ::= FLOAT", - /* 138 */ "expr ::= MINUS FLOAT", - /* 139 */ "expr ::= PLUS FLOAT", - /* 140 */ "expr ::= STRING", - /* 141 */ "expr ::= NOW", - /* 142 */ "expr ::= VARIABLE", - /* 143 */ "expr ::= BOOL", - /* 144 */ "expr ::= ID LP exprlist RP", - /* 145 */ "expr ::= ID LP STAR RP", - /* 146 */ "expr ::= expr AND expr", - /* 147 */ "expr ::= expr OR expr", - /* 148 */ "expr ::= expr LT expr", - /* 149 */ "expr ::= expr GT expr", - /* 150 */ "expr ::= expr LE expr", - /* 151 */ "expr ::= expr GE expr", - /* 152 */ "expr ::= expr NE expr", - /* 153 */ "expr ::= expr EQ expr", - /* 154 */ "expr ::= expr PLUS expr", - /* 155 */ "expr ::= expr MINUS expr", - /* 156 */ "expr ::= expr STAR expr", - /* 157 */ "expr ::= expr SLASH expr", - /* 158 */ "expr ::= expr REM expr", - /* 159 */ "expr ::= expr LIKE expr", - /* 160 */ "expr ::= expr IN LP exprlist RP", - /* 161 */ "exprlist ::= exprlist COMMA expritem", - /* 162 */ "exprlist ::= expritem", - /* 163 */ "expritem ::= expr", - /* 164 */ "expritem ::=", - /* 165 */ "cmd ::= INSERT INTO cpxName insert_value_list", - /* 166 */ "insert_value_list ::= VALUES LP itemlist RP", - /* 167 */ "insert_value_list ::= insert_value_list VALUES LP itemlist RP", - /* 168 */ "itemlist ::= itemlist COMMA expr", - /* 169 */ "itemlist ::= expr", - /* 170 */ "cmd ::= RESET QUERY CACHE", - /* 171 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 172 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 173 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 174 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 175 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 176 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 177 */ "cmd ::= KILL CONNECTION IP COLON INTEGER", - /* 178 */ "cmd ::= KILL STREAM IP COLON INTEGER COLON INTEGER", - /* 179 */ "cmd ::= KILL QUERY IP COLON INTEGER COLON INTEGER", + /* 4 */ "cmd ::= SHOW ACCOUNTS", + /* 5 */ "cmd ::= SHOW USERS", + /* 6 */ "cmd ::= SHOW MODULES", + /* 7 */ "cmd ::= SHOW QUERIES", + /* 8 */ "cmd ::= SHOW CONNECTIONS", + /* 9 */ "cmd ::= SHOW STREAMS", + /* 10 */ "cmd ::= SHOW CONFIGS", + /* 11 */ "cmd ::= SHOW SCORES", + /* 12 */ "cmd ::= SHOW GRANTS", + /* 13 */ "dbPrefix ::=", + /* 14 */ "dbPrefix ::= ids DOT", + /* 15 */ "cpxName ::=", + /* 16 */ "cpxName ::= DOT ids", + /* 17 */ "cmd ::= SHOW dbPrefix TABLES", + /* 18 */ "cmd ::= SHOW dbPrefix TABLES LIKE ids", + /* 19 */ "cmd ::= SHOW dbPrefix STABLES", + /* 20 */ "cmd ::= SHOW dbPrefix STABLES LIKE ids", + /* 21 */ "cmd ::= SHOW dbPrefix VGROUPS", + /* 22 */ "cmd ::= DROP TABLE ifexists ids cpxName", + /* 23 */ "cmd ::= DROP DATABASE ifexists ids", + /* 24 */ "cmd ::= DROP DNODE IP", + /* 25 */ "cmd ::= DROP USER ids", + /* 26 */ "cmd ::= DROP ACCOUNT ids", + /* 27 */ "cmd ::= USE ids", + /* 28 */ "cmd ::= DESCRIBE ids cpxName", + /* 29 */ "cmd ::= ALTER USER ids PASS ids", + /* 30 */ "cmd ::= ALTER USER ids PRIVILEGE ids", + /* 31 */ "cmd ::= ALTER DNODE IP ids", + /* 32 */ "cmd ::= ALTER DNODE IP ids ids", + /* 33 */ "cmd ::= ALTER LOCAL ids", + /* 34 */ "cmd ::= ALTER LOCAL ids ids", + /* 35 */ "cmd ::= ALTER DATABASE ids alter_db_optr", + /* 36 */ "cmd ::= ALTER ACCOUNT ids acct_optr", + /* 37 */ "cmd ::= ALTER ACCOUNT ids PASS ids acct_optr", + /* 38 */ "ids ::= ID", + /* 39 */ "ids ::= STRING", + /* 40 */ "ifexists ::= IF EXISTS", + /* 41 */ "ifexists ::=", + /* 42 */ "ifnotexists ::= IF NOT EXISTS", + /* 43 */ "ifnotexists ::=", + /* 44 */ "cmd ::= CREATE DNODE IP", + /* 45 */ "cmd ::= CREATE ACCOUNT ids PASS ids acct_optr", + /* 46 */ "cmd ::= CREATE DATABASE ifnotexists ids db_optr", + /* 47 */ "cmd ::= CREATE USER ids PASS ids", + /* 48 */ "pps ::=", + /* 49 */ "pps ::= PPS INTEGER", + /* 50 */ "tseries ::=", + /* 51 */ "tseries ::= TSERIES INTEGER", + /* 52 */ "dbs ::=", + /* 53 */ "dbs ::= DBS INTEGER", + /* 54 */ "streams ::=", + /* 55 */ "streams ::= STREAMS INTEGER", + /* 56 */ "storage ::=", + /* 57 */ "storage ::= STORAGE INTEGER", + /* 58 */ "qtime ::=", + /* 59 */ "qtime ::= QTIME INTEGER", + /* 60 */ "users ::=", + /* 61 */ "users ::= USERS INTEGER", + /* 62 */ "conns ::=", + /* 63 */ "conns ::= CONNS INTEGER", + /* 64 */ "state ::=", + /* 65 */ "state ::= STATE ids", + /* 66 */ "acct_optr ::= pps tseries storage streams qtime dbs users conns state", + /* 67 */ "keep ::= KEEP tagitemlist", + /* 68 */ "tables ::= TABLES INTEGER", + /* 69 */ "cache ::= CACHE INTEGER", + /* 70 */ "replica ::= REPLICA INTEGER", + /* 71 */ "days ::= DAYS INTEGER", + /* 72 */ "rows ::= ROWS INTEGER", + /* 73 */ "ablocks ::= ABLOCKS ID", + /* 74 */ "tblocks ::= TBLOCKS INTEGER", + /* 75 */ "ctime ::= CTIME INTEGER", + /* 76 */ "clog ::= CLOG INTEGER", + /* 77 */ "comp ::= COMP INTEGER", + /* 78 */ "prec ::= PRECISION STRING", + /* 79 */ "db_optr ::=", + /* 80 */ "db_optr ::= db_optr tables", + /* 81 */ "db_optr ::= db_optr cache", + /* 82 */ "db_optr ::= db_optr replica", + /* 83 */ "db_optr ::= db_optr days", + /* 84 */ "db_optr ::= db_optr rows", + /* 85 */ "db_optr ::= db_optr ablocks", + /* 86 */ "db_optr ::= db_optr tblocks", + /* 87 */ "db_optr ::= db_optr ctime", + /* 88 */ "db_optr ::= db_optr clog", + /* 89 */ "db_optr ::= db_optr comp", + /* 90 */ "db_optr ::= db_optr prec", + /* 91 */ "db_optr ::= db_optr keep", + /* 92 */ "alter_db_optr ::=", + /* 93 */ "alter_db_optr ::= alter_db_optr replica", + /* 94 */ "alter_db_optr ::= alter_db_optr tables", + /* 95 */ "typename ::= ids", + /* 96 */ "typename ::= ids LP signed RP", + /* 97 */ "signed ::= INTEGER", + /* 98 */ "signed ::= PLUS INTEGER", + /* 99 */ "signed ::= MINUS INTEGER", + /* 100 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args", + /* 101 */ "create_table_args ::= LP columnlist RP", + /* 102 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP", + /* 103 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP", + /* 104 */ "create_table_args ::= AS select", + /* 105 */ "columnlist ::= columnlist COMMA column", + /* 106 */ "columnlist ::= column", + /* 107 */ "column ::= ids typename", + /* 108 */ "tagitemlist ::= tagitemlist COMMA tagitem", + /* 109 */ "tagitemlist ::= tagitem", + /* 110 */ "tagitem ::= INTEGER", + /* 111 */ "tagitem ::= FLOAT", + /* 112 */ "tagitem ::= STRING", + /* 113 */ "tagitem ::= BOOL", + /* 114 */ "tagitem ::= NULL", + /* 115 */ "tagitem ::= MINUS INTEGER", + /* 116 */ "tagitem ::= MINUS FLOAT", + /* 117 */ "tagitem ::= PLUS INTEGER", + /* 118 */ "tagitem ::= PLUS FLOAT", + /* 119 */ "cmd ::= select", + /* 120 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", + /* 121 */ "sclp ::= selcollist COMMA", + /* 122 */ "sclp ::=", + /* 123 */ "selcollist ::= sclp expr as", + /* 124 */ "selcollist ::= sclp STAR", + /* 125 */ "as ::= AS ids", + /* 126 */ "as ::= ids", + /* 127 */ "as ::=", + /* 128 */ "from ::= FROM tablelist", + /* 129 */ "tablelist ::= ids cpxName", + /* 130 */ "tablelist ::= tablelist COMMA ids cpxName", + /* 131 */ "tmvar ::= VARIABLE", + /* 132 */ "interval_opt ::= INTERVAL LP tmvar RP", + /* 133 */ "interval_opt ::=", + /* 134 */ "fill_opt ::=", + /* 135 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", + /* 136 */ "fill_opt ::= FILL LP ID RP", + /* 137 */ "sliding_opt ::= SLIDING LP tmvar RP", + /* 138 */ "sliding_opt ::=", + /* 139 */ "orderby_opt ::=", + /* 140 */ "orderby_opt ::= ORDER BY sortlist", + /* 141 */ "sortlist ::= sortlist COMMA item sortorder", + /* 142 */ "sortlist ::= item sortorder", + /* 143 */ "item ::= ids cpxName", + /* 144 */ "sortorder ::= ASC", + /* 145 */ "sortorder ::= DESC", + /* 146 */ "sortorder ::=", + /* 147 */ "groupby_opt ::=", + /* 148 */ "groupby_opt ::= GROUP BY grouplist", + /* 149 */ "grouplist ::= grouplist COMMA item", + /* 150 */ "grouplist ::= item", + /* 151 */ "having_opt ::=", + /* 152 */ "having_opt ::= HAVING expr", + /* 153 */ "limit_opt ::=", + /* 154 */ "limit_opt ::= LIMIT signed", + /* 155 */ "limit_opt ::= LIMIT signed OFFSET signed", + /* 156 */ "limit_opt ::= LIMIT signed COMMA signed", + /* 157 */ "slimit_opt ::=", + /* 158 */ "slimit_opt ::= SLIMIT signed", + /* 159 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", + /* 160 */ "slimit_opt ::= SLIMIT signed COMMA signed", + /* 161 */ "where_opt ::=", + /* 162 */ "where_opt ::= WHERE expr", + /* 163 */ "expr ::= LP expr RP", + /* 164 */ "expr ::= ID", + /* 165 */ "expr ::= ID DOT ID", + /* 166 */ "expr ::= ID DOT STAR", + /* 167 */ "expr ::= INTEGER", + /* 168 */ "expr ::= MINUS INTEGER", + /* 169 */ "expr ::= PLUS INTEGER", + /* 170 */ "expr ::= FLOAT", + /* 171 */ "expr ::= MINUS FLOAT", + /* 172 */ "expr ::= PLUS FLOAT", + /* 173 */ "expr ::= STRING", + /* 174 */ "expr ::= NOW", + /* 175 */ "expr ::= VARIABLE", + /* 176 */ "expr ::= BOOL", + /* 177 */ "expr ::= ID LP exprlist RP", + /* 178 */ "expr ::= ID LP STAR RP", + /* 179 */ "expr ::= expr AND expr", + /* 180 */ "expr ::= expr OR expr", + /* 181 */ "expr ::= expr LT expr", + /* 182 */ "expr ::= expr GT expr", + /* 183 */ "expr ::= expr LE expr", + /* 184 */ "expr ::= expr GE expr", + /* 185 */ "expr ::= expr NE expr", + /* 186 */ "expr ::= expr EQ expr", + /* 187 */ "expr ::= expr PLUS expr", + /* 188 */ "expr ::= expr MINUS expr", + /* 189 */ "expr ::= expr STAR expr", + /* 190 */ "expr ::= expr SLASH expr", + /* 191 */ "expr ::= expr REM expr", + /* 192 */ "expr ::= expr LIKE expr", + /* 193 */ "expr ::= expr IN LP exprlist RP", + /* 194 */ "exprlist ::= exprlist COMMA expritem", + /* 195 */ "exprlist ::= expritem", + /* 196 */ "expritem ::= expr", + /* 197 */ "expritem ::=", + /* 198 */ "cmd ::= INSERT INTO cpxName insert_value_list", + /* 199 */ "insert_value_list ::= VALUES LP itemlist RP", + /* 200 */ "insert_value_list ::= insert_value_list VALUES LP itemlist RP", + /* 201 */ "itemlist ::= itemlist COMMA expr", + /* 202 */ "itemlist ::= expr", + /* 203 */ "cmd ::= RESET QUERY CACHE", + /* 204 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 205 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 206 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 207 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 208 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 209 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 210 */ "cmd ::= KILL CONNECTION IP COLON INTEGER", + /* 211 */ "cmd ::= KILL STREAM IP COLON INTEGER COLON INTEGER", + /* 212 */ "cmd ::= KILL QUERY IP COLON INTEGER COLON INTEGER", }; #endif /* NDEBUG */ @@ -995,46 +1063,46 @@ static void yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 196: /* keep */ - case 197: /* tagitemlist */ - case 220: /* fill_opt */ - case 222: /* groupby_opt */ - case 223: /* orderby_opt */ - case 231: /* sortlist */ - case 235: /* grouplist */ + case 215: /* keep */ + case 216: /* tagitemlist */ + case 239: /* fill_opt */ + case 241: /* groupby_opt */ + case 242: /* orderby_opt */ + case 251: /* sortlist */ + case 255: /* grouplist */ { -tVariantListDestroy((yypminor->yy216)); +tVariantListDestroy((yypminor->yy56)); } break; - case 213: /* columnlist */ + case 231: /* columnlist */ { -tFieldListDestroy((yypminor->yy151)); +tFieldListDestroy((yypminor->yy471)); } break; - case 214: /* select */ + case 232: /* select */ { destroyQuerySql((yypminor->yy24)); } break; - case 216: /* selcollist */ - case 227: /* sclp */ - case 236: /* exprlist */ - case 239: /* itemlist */ + case 235: /* selcollist */ + case 246: /* sclp */ + case 256: /* exprlist */ + case 259: /* itemlist */ { -tSQLExprListDestroy((yypminor->yy98)); +tSQLExprListDestroy((yypminor->yy498)); } break; - case 218: /* where_opt */ - case 224: /* having_opt */ - case 228: /* expr */ - case 237: /* expritem */ + case 237: /* where_opt */ + case 243: /* having_opt */ + case 247: /* expr */ + case 257: /* expritem */ { -tSQLExprDestroy((yypminor->yy370)); +tSQLExprDestroy((yypminor->yy90)); } break; - case 232: /* sortitem */ + case 252: /* sortitem */ { -tVariantDestroy(&(yypminor->yy266)); +tVariantDestroy(&(yypminor->yy186)); } break; /********* End destructor definitions *****************************************/ @@ -1275,186 +1343,219 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ unsigned char nrhs; /* Number of right-hand side symbols in the rule */ } yyRuleInfo[] = { - { 187, 1 }, - { 188, 2 }, - { 188, 2 }, - { 188, 2 }, - { 188, 2 }, - { 188, 2 }, - { 188, 2 }, - { 188, 2 }, - { 188, 2 }, - { 188, 2 }, - { 188, 2 }, - { 188, 2 }, - { 189, 0 }, - { 189, 2 }, - { 191, 0 }, - { 191, 2 }, - { 188, 3 }, - { 188, 5 }, - { 188, 3 }, - { 188, 5 }, - { 188, 3 }, - { 188, 5 }, - { 188, 4 }, - { 188, 3 }, - { 188, 2 }, - { 188, 3 }, - { 188, 5 }, - { 188, 5 }, - { 188, 4 }, - { 188, 5 }, - { 188, 3 }, - { 188, 4 }, - { 190, 1 }, - { 190, 1 }, - { 192, 2 }, - { 192, 0 }, - { 194, 3 }, - { 194, 0 }, - { 188, 5 }, - { 188, 5 }, - { 196, 2 }, + { 196, 1 }, + { 197, 2 }, + { 197, 2 }, + { 197, 2 }, + { 197, 2 }, + { 197, 2 }, + { 197, 2 }, + { 197, 2 }, + { 197, 2 }, + { 197, 2 }, + { 197, 2 }, + { 197, 2 }, + { 197, 2 }, + { 198, 0 }, { 198, 2 }, - { 199, 2 }, + { 200, 0 }, { 200, 2 }, + { 197, 3 }, + { 197, 5 }, + { 197, 3 }, + { 197, 5 }, + { 197, 3 }, + { 197, 5 }, + { 197, 4 }, + { 197, 3 }, + { 197, 3 }, + { 197, 3 }, + { 197, 2 }, + { 197, 3 }, + { 197, 5 }, + { 197, 5 }, + { 197, 4 }, + { 197, 5 }, + { 197, 3 }, + { 197, 4 }, + { 197, 4 }, + { 197, 4 }, + { 197, 6 }, + { 199, 1 }, + { 199, 1 }, { 201, 2 }, - { 202, 2 }, - { 203, 2 }, - { 204, 2 }, - { 205, 2 }, + { 201, 0 }, + { 204, 3 }, + { 204, 0 }, + { 197, 3 }, + { 197, 6 }, + { 197, 5 }, + { 197, 5 }, + { 206, 0 }, { 206, 2 }, + { 207, 0 }, { 207, 2 }, + { 208, 0 }, { 208, 2 }, - { 195, 0 }, - { 195, 2 }, - { 195, 2 }, - { 195, 2 }, - { 195, 2 }, - { 195, 2 }, - { 195, 2 }, - { 195, 2 }, - { 195, 2 }, - { 195, 2 }, - { 195, 2 }, - { 195, 2 }, - { 195, 2 }, - { 193, 2 }, - { 210, 1 }, - { 210, 4 }, - { 211, 1 }, - { 211, 2 }, + { 209, 0 }, + { 209, 2 }, + { 210, 0 }, + { 210, 2 }, + { 211, 0 }, { 211, 2 }, - { 188, 6 }, - { 212, 3 }, - { 212, 7 }, - { 212, 7 }, + { 212, 0 }, { 212, 2 }, - { 213, 3 }, - { 213, 1 }, + { 213, 0 }, + { 213, 2 }, + { 214, 0 }, + { 214, 2 }, + { 203, 9 }, { 215, 2 }, - { 197, 3 }, - { 197, 1 }, - { 209, 1 }, - { 209, 1 }, - { 209, 1 }, - { 209, 1 }, - { 209, 1 }, - { 209, 2 }, - { 209, 2 }, - { 188, 1 }, - { 214, 12 }, + { 217, 2 }, + { 218, 2 }, + { 219, 2 }, + { 220, 2 }, + { 221, 2 }, + { 222, 2 }, + { 223, 2 }, + { 224, 2 }, + { 225, 2 }, + { 226, 2 }, { 227, 2 }, - { 227, 0 }, - { 216, 3 }, - { 216, 2 }, - { 229, 2 }, + { 205, 0 }, + { 205, 2 }, + { 205, 2 }, + { 205, 2 }, + { 205, 2 }, + { 205, 2 }, + { 205, 2 }, + { 205, 2 }, + { 205, 2 }, + { 205, 2 }, + { 205, 2 }, + { 205, 2 }, + { 205, 2 }, + { 202, 0 }, + { 202, 2 }, + { 202, 2 }, + { 228, 1 }, + { 228, 4 }, { 229, 1 }, - { 229, 0 }, - { 217, 3 }, - { 230, 1 }, - { 219, 4 }, - { 219, 0 }, - { 220, 0 }, - { 220, 6 }, - { 220, 4 }, - { 221, 4 }, - { 221, 0 }, - { 223, 0 }, - { 223, 3 }, - { 231, 4 }, - { 231, 2 }, + { 229, 2 }, + { 229, 2 }, + { 197, 6 }, + { 230, 3 }, + { 230, 7 }, + { 230, 7 }, + { 230, 2 }, + { 231, 3 }, + { 231, 1 }, { 233, 2 }, + { 216, 3 }, + { 216, 1 }, + { 234, 1 }, + { 234, 1 }, + { 234, 1 }, { 234, 1 }, { 234, 1 }, - { 234, 0 }, - { 222, 0 }, - { 222, 3 }, + { 234, 2 }, + { 234, 2 }, + { 234, 2 }, + { 234, 2 }, + { 197, 1 }, + { 232, 12 }, + { 246, 2 }, + { 246, 0 }, { 235, 3 }, - { 235, 1 }, - { 224, 0 }, - { 224, 2 }, - { 226, 0 }, - { 226, 2 }, - { 226, 4 }, - { 226, 4 }, - { 225, 0 }, - { 225, 2 }, - { 225, 4 }, - { 225, 4 }, - { 218, 0 }, - { 218, 2 }, - { 228, 3 }, - { 228, 1 }, - { 228, 3 }, - { 228, 3 }, - { 228, 1 }, - { 228, 2 }, - { 228, 2 }, - { 228, 1 }, - { 228, 2 }, - { 228, 2 }, - { 228, 1 }, - { 228, 1 }, - { 228, 1 }, - { 228, 1 }, - { 228, 4 }, - { 228, 4 }, - { 228, 3 }, - { 228, 3 }, - { 228, 3 }, - { 228, 3 }, - { 228, 3 }, - { 228, 3 }, - { 228, 3 }, - { 228, 3 }, - { 228, 3 }, - { 228, 3 }, - { 228, 3 }, - { 228, 3 }, - { 228, 3 }, - { 228, 3 }, - { 228, 5 }, - { 236, 3 }, - { 236, 1 }, - { 237, 1 }, - { 237, 0 }, - { 188, 4 }, + { 235, 2 }, + { 248, 2 }, + { 248, 1 }, + { 248, 0 }, + { 236, 2 }, + { 249, 2 }, + { 249, 4 }, + { 250, 1 }, { 238, 4 }, - { 238, 5 }, - { 239, 3 }, - { 239, 1 }, - { 188, 3 }, - { 188, 7 }, - { 188, 7 }, - { 188, 7 }, - { 188, 7 }, - { 188, 8 }, - { 188, 9 }, - { 188, 5 }, - { 188, 7 }, - { 188, 7 }, + { 238, 0 }, + { 239, 0 }, + { 239, 6 }, + { 239, 4 }, + { 240, 4 }, + { 240, 0 }, + { 242, 0 }, + { 242, 3 }, + { 251, 4 }, + { 251, 2 }, + { 253, 2 }, + { 254, 1 }, + { 254, 1 }, + { 254, 0 }, + { 241, 0 }, + { 241, 3 }, + { 255, 3 }, + { 255, 1 }, + { 243, 0 }, + { 243, 2 }, + { 245, 0 }, + { 245, 2 }, + { 245, 4 }, + { 245, 4 }, + { 244, 0 }, + { 244, 2 }, + { 244, 4 }, + { 244, 4 }, + { 237, 0 }, + { 237, 2 }, + { 247, 3 }, + { 247, 1 }, + { 247, 3 }, + { 247, 3 }, + { 247, 1 }, + { 247, 2 }, + { 247, 2 }, + { 247, 1 }, + { 247, 2 }, + { 247, 2 }, + { 247, 1 }, + { 247, 1 }, + { 247, 1 }, + { 247, 1 }, + { 247, 4 }, + { 247, 4 }, + { 247, 3 }, + { 247, 3 }, + { 247, 3 }, + { 247, 3 }, + { 247, 3 }, + { 247, 3 }, + { 247, 3 }, + { 247, 3 }, + { 247, 3 }, + { 247, 3 }, + { 247, 3 }, + { 247, 3 }, + { 247, 3 }, + { 247, 3 }, + { 247, 5 }, + { 256, 3 }, + { 256, 1 }, + { 257, 1 }, + { 257, 0 }, + { 197, 4 }, + { 258, 4 }, + { 258, 5 }, + { 259, 3 }, + { 259, 1 }, + { 197, 3 }, + { 197, 7 }, + { 197, 7 }, + { 197, 7 }, + { 197, 7 }, + { 197, 8 }, + { 197, 9 }, + { 197, 5 }, + { 197, 7 }, + { 197, 7 }, }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -1506,527 +1607,590 @@ static void yy_reduce( case 3: /* cmd ::= SHOW DNODES */ { setDCLSQLElems(pInfo, SHOW_DNODES, 0);} break; - case 4: /* cmd ::= SHOW USERS */ + case 4: /* cmd ::= SHOW ACCOUNTS */ +{ setDCLSQLElems(pInfo, SHOW_ACCOUNTS, 0);} + break; + case 5: /* cmd ::= SHOW USERS */ { setDCLSQLElems(pInfo, SHOW_USERS, 0);} break; - case 5: /* cmd ::= SHOW MODULES */ + case 6: /* cmd ::= SHOW MODULES */ { setDCLSQLElems(pInfo, SHOW_MODULES, 0); } break; - case 6: /* cmd ::= SHOW QUERIES */ + case 7: /* cmd ::= SHOW QUERIES */ { setDCLSQLElems(pInfo, SHOW_QUERIES, 0); } break; - case 7: /* cmd ::= SHOW CONNECTIONS */ + case 8: /* cmd ::= SHOW CONNECTIONS */ { setDCLSQLElems(pInfo, SHOW_CONNECTIONS, 0);} break; - case 8: /* cmd ::= SHOW STREAMS */ + case 9: /* cmd ::= SHOW STREAMS */ { setDCLSQLElems(pInfo, SHOW_STREAMS, 0); } break; - case 9: /* cmd ::= SHOW CONFIGS */ + case 10: /* cmd ::= SHOW CONFIGS */ { setDCLSQLElems(pInfo, SHOW_CONFIGS, 0); } break; - case 10: /* cmd ::= SHOW SCORES */ + case 11: /* cmd ::= SHOW SCORES */ { setDCLSQLElems(pInfo, SHOW_SCORES, 0); } break; - case 11: /* cmd ::= SHOW GRANTS */ + case 12: /* cmd ::= SHOW GRANTS */ { setDCLSQLElems(pInfo, SHOW_GRANTS, 0); } break; - case 12: /* dbPrefix ::= */ - case 35: /* ifexists ::= */ yytestcase(yyruleno==35); - case 37: /* ifnotexists ::= */ yytestcase(yyruleno==37); + case 13: /* dbPrefix ::= */ + case 41: /* ifexists ::= */ yytestcase(yyruleno==41); + case 43: /* ifnotexists ::= */ yytestcase(yyruleno==43); {yygotominor.yy0.n = 0;} break; - case 13: /* dbPrefix ::= ids DOT */ + case 14: /* dbPrefix ::= ids DOT */ {yygotominor.yy0 = yymsp[-1].minor.yy0; } break; - case 14: /* cpxName ::= */ + case 15: /* cpxName ::= */ {yygotominor.yy0.n = 0; } break; - case 15: /* cpxName ::= DOT ids */ + case 16: /* cpxName ::= DOT ids */ {yygotominor.yy0 = yymsp[0].minor.yy0; yygotominor.yy0.n += 1; } break; - case 16: /* cmd ::= SHOW dbPrefix TABLES */ + case 17: /* cmd ::= SHOW dbPrefix TABLES */ { setDCLSQLElems(pInfo, SHOW_TABLES, 1, &yymsp[-1].minor.yy0); } break; - case 17: /* cmd ::= SHOW dbPrefix TABLES LIKE ids */ + case 18: /* cmd ::= SHOW dbPrefix TABLES LIKE ids */ { setDCLSQLElems(pInfo, SHOW_TABLES, 2, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0); } break; - case 18: /* cmd ::= SHOW dbPrefix STABLES */ + case 19: /* cmd ::= SHOW dbPrefix STABLES */ { setDCLSQLElems(pInfo, SHOW_STABLES, 1, &yymsp[-1].minor.yy0); } break; - case 19: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ + case 20: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ { SSQLToken token; setDBName(&token, &yymsp[-3].minor.yy0); setDCLSQLElems(pInfo, SHOW_STABLES, 2, &token, &yymsp[0].minor.yy0); } break; - case 20: /* cmd ::= SHOW dbPrefix VGROUPS */ + case 21: /* cmd ::= SHOW dbPrefix VGROUPS */ { SSQLToken token; setDBName(&token, &yymsp[-1].minor.yy0); setDCLSQLElems(pInfo, SHOW_VGROUPS, 1, &token); } break; - case 21: /* cmd ::= DROP TABLE ifexists ids cpxName */ + case 22: /* cmd ::= DROP TABLE ifexists ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; setDCLSQLElems(pInfo, DROP_TABLE, 2, &yymsp[-1].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 22: /* cmd ::= DROP DATABASE ifexists ids */ + case 23: /* cmd ::= DROP DATABASE ifexists ids */ { setDCLSQLElems(pInfo, DROP_DATABASE, 2, &yymsp[0].minor.yy0, &yymsp[-1].minor.yy0); } break; - case 23: /* cmd ::= DROP USER ids */ + case 24: /* cmd ::= DROP DNODE IP */ +{ setDCLSQLElems(pInfo, DROP_DNODE, 1, &yymsp[0].minor.yy0); } + break; + case 25: /* cmd ::= DROP USER ids */ { setDCLSQLElems(pInfo, DROP_USER, 1, &yymsp[0].minor.yy0); } break; - case 24: /* cmd ::= USE ids */ + case 26: /* cmd ::= DROP ACCOUNT ids */ +{ setDCLSQLElems(pInfo, DROP_ACCOUNT, 1, &yymsp[0].minor.yy0); } + break; + case 27: /* cmd ::= USE ids */ { setDCLSQLElems(pInfo, USE_DATABASE, 1, &yymsp[0].minor.yy0);} break; - case 25: /* cmd ::= DESCRIBE ids cpxName */ + case 28: /* cmd ::= DESCRIBE ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; setDCLSQLElems(pInfo, DESCRIBE_TABLE, 1, &yymsp[-1].minor.yy0); } break; - case 26: /* cmd ::= ALTER USER ids PASS ids */ + case 29: /* cmd ::= ALTER USER ids PASS ids */ { setDCLSQLElems(pInfo, ALTER_USER_PASSWD, 2, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 27: /* cmd ::= ALTER USER ids PRIVILEGE ids */ + case 30: /* cmd ::= ALTER USER ids PRIVILEGE ids */ { setDCLSQLElems(pInfo, ALTER_USER_PRIVILEGES, 2, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} break; - case 28: /* cmd ::= ALTER DNODE IP ids */ + case 31: /* cmd ::= ALTER DNODE IP ids */ { setDCLSQLElems(pInfo, ALTER_DNODE, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 29: /* cmd ::= ALTER DNODE IP ids ids */ + case 32: /* cmd ::= ALTER DNODE IP ids ids */ { setDCLSQLElems(pInfo, ALTER_DNODE, 3, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 30: /* cmd ::= ALTER LOCAL ids */ + case 33: /* cmd ::= ALTER LOCAL ids */ { setDCLSQLElems(pInfo, ALTER_LOCAL, 1, &yymsp[0].minor.yy0); } break; - case 31: /* cmd ::= ALTER DATABASE ids alter_db_optr */ -{ SSQLToken t = {0}; setCreateDBSQL(pInfo, ALTER_DATABASE, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy478, &t);} + case 34: /* cmd ::= ALTER LOCAL ids ids */ +{ setDCLSQLElems(pInfo, ALTER_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } + break; + case 35: /* cmd ::= ALTER DATABASE ids alter_db_optr */ +{ SSQLToken t = {0}; setCreateDBSQL(pInfo, ALTER_DATABASE, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy398, &t);} + break; + case 36: /* cmd ::= ALTER ACCOUNT ids acct_optr */ +{ SSQLToken t = {0}; setCreateAcctSQL(pInfo, ALTER_ACCT, &yymsp[-1].minor.yy0, &t, &yymsp[0].minor.yy279);} break; - case 32: /* ids ::= ID */ - case 33: /* ids ::= STRING */ yytestcase(yyruleno==33); + case 37: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ +{ setCreateAcctSQL(pInfo, ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy279);} + break; + case 38: /* ids ::= ID */ + case 39: /* ids ::= STRING */ yytestcase(yyruleno==39); {yygotominor.yy0 = yymsp[0].minor.yy0; } break; - case 34: /* ifexists ::= IF EXISTS */ - case 36: /* ifnotexists ::= IF NOT EXISTS */ yytestcase(yyruleno==36); + case 40: /* ifexists ::= IF EXISTS */ + case 42: /* ifnotexists ::= IF NOT EXISTS */ yytestcase(yyruleno==42); {yygotominor.yy0.n = 1;} break; - case 38: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ -{ setCreateDBSQL(pInfo, CREATE_DATABASE, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy478, &yymsp[-2].minor.yy0);} + case 44: /* cmd ::= CREATE DNODE IP */ +{ setDCLSQLElems(pInfo, CREATE_DNODE, 1, &yymsp[0].minor.yy0);} + break; + case 45: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ +{ setCreateAcctSQL(pInfo, CREATE_ACCOUNT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy279);} + break; + case 46: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ +{ setCreateDBSQL(pInfo, CREATE_DATABASE, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy398, &yymsp[-2].minor.yy0);} break; - case 39: /* cmd ::= CREATE USER ids PASS ids */ + case 47: /* cmd ::= CREATE USER ids PASS ids */ { setDCLSQLElems(pInfo, CREATE_USER, 2, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} break; - case 40: /* keep ::= KEEP tagitemlist */ -{ yygotominor.yy216 = yymsp[0].minor.yy216; } - break; - case 41: /* tables ::= TABLES INTEGER */ - case 42: /* cache ::= CACHE INTEGER */ yytestcase(yyruleno==42); - case 43: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==43); - case 44: /* days ::= DAYS INTEGER */ yytestcase(yyruleno==44); - case 45: /* rows ::= ROWS INTEGER */ yytestcase(yyruleno==45); - case 46: /* ablocks ::= ABLOCKS ID */ yytestcase(yyruleno==46); - case 47: /* tblocks ::= TBLOCKS INTEGER */ yytestcase(yyruleno==47); - case 48: /* ctime ::= CTIME INTEGER */ yytestcase(yyruleno==48); - case 49: /* clog ::= CLOG INTEGER */ yytestcase(yyruleno==49); - case 50: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==50); - case 51: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==51); - case 52: /* db_optr ::= */ yytestcase(yyruleno==52); + case 48: /* pps ::= */ + case 50: /* tseries ::= */ yytestcase(yyruleno==50); + case 52: /* dbs ::= */ yytestcase(yyruleno==52); + case 54: /* streams ::= */ yytestcase(yyruleno==54); + case 56: /* storage ::= */ yytestcase(yyruleno==56); + case 58: /* qtime ::= */ yytestcase(yyruleno==58); + case 60: /* users ::= */ yytestcase(yyruleno==60); + case 62: /* conns ::= */ yytestcase(yyruleno==62); + case 64: /* state ::= */ yytestcase(yyruleno==64); + case 133: /* interval_opt ::= */ yytestcase(yyruleno==133); + case 138: /* sliding_opt ::= */ yytestcase(yyruleno==138); +{yygotominor.yy0.n = 0; } + break; + case 49: /* pps ::= PPS INTEGER */ + case 51: /* tseries ::= TSERIES INTEGER */ yytestcase(yyruleno==51); + case 53: /* dbs ::= DBS INTEGER */ yytestcase(yyruleno==53); + case 55: /* streams ::= STREAMS INTEGER */ yytestcase(yyruleno==55); + case 57: /* storage ::= STORAGE INTEGER */ yytestcase(yyruleno==57); + case 59: /* qtime ::= QTIME INTEGER */ yytestcase(yyruleno==59); + case 61: /* users ::= USERS INTEGER */ yytestcase(yyruleno==61); + case 63: /* conns ::= CONNS INTEGER */ yytestcase(yyruleno==63); + case 65: /* state ::= STATE ids */ yytestcase(yyruleno==65); +{yygotominor.yy0 = yymsp[0].minor.yy0; } + break; + case 66: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ +{ + yygotominor.yy279.users = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; + yygotominor.yy279.dbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; + yygotominor.yy279.tseries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; + yygotominor.yy279.streams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; + yygotominor.yy279.pps = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; + yygotominor.yy279.storage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; + yygotominor.yy279.qtime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; + yygotominor.yy279.conns = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; + yygotominor.yy279.stat = yymsp[0].minor.yy0; +} + break; + case 67: /* keep ::= KEEP tagitemlist */ +{ yygotominor.yy56 = yymsp[0].minor.yy56; } + break; + case 68: /* tables ::= TABLES INTEGER */ + case 69: /* cache ::= CACHE INTEGER */ yytestcase(yyruleno==69); + case 70: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==70); + case 71: /* days ::= DAYS INTEGER */ yytestcase(yyruleno==71); + case 72: /* rows ::= ROWS INTEGER */ yytestcase(yyruleno==72); + case 73: /* ablocks ::= ABLOCKS ID */ yytestcase(yyruleno==73); + case 74: /* tblocks ::= TBLOCKS INTEGER */ yytestcase(yyruleno==74); + case 75: /* ctime ::= CTIME INTEGER */ yytestcase(yyruleno==75); + case 76: /* clog ::= CLOG INTEGER */ yytestcase(yyruleno==76); + case 77: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==77); + case 78: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==78); + case 79: /* db_optr ::= */ yytestcase(yyruleno==79); { yygotominor.yy0 = yymsp[0].minor.yy0; } break; - case 53: /* db_optr ::= db_optr tables */ -{ yygotominor.yy478 = yymsp[-1].minor.yy478; yygotominor.yy478.tablesPerVnode = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 80: /* db_optr ::= db_optr tables */ + case 94: /* alter_db_optr ::= alter_db_optr tables */ yytestcase(yyruleno==94); +{ yygotominor.yy398 = yymsp[-1].minor.yy398; yygotominor.yy398.tablesPerVnode = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 54: /* db_optr ::= db_optr cache */ -{ yygotominor.yy478 = yymsp[-1].minor.yy478; yygotominor.yy478.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 81: /* db_optr ::= db_optr cache */ +{ yygotominor.yy398 = yymsp[-1].minor.yy398; yygotominor.yy398.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 55: /* db_optr ::= db_optr replica */ -{ yygotominor.yy478 = yymsp[-1].minor.yy478; yygotominor.yy478.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 82: /* db_optr ::= db_optr replica */ + case 93: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==93); +{ yygotominor.yy398 = yymsp[-1].minor.yy398; yygotominor.yy398.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 56: /* db_optr ::= db_optr days */ -{ yygotominor.yy478 = yymsp[-1].minor.yy478; yygotominor.yy478.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 83: /* db_optr ::= db_optr days */ +{ yygotominor.yy398 = yymsp[-1].minor.yy398; yygotominor.yy398.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 57: /* db_optr ::= db_optr rows */ -{ yygotominor.yy478 = yymsp[-1].minor.yy478; yygotominor.yy478.rowPerFileBlock = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 84: /* db_optr ::= db_optr rows */ +{ yygotominor.yy398 = yymsp[-1].minor.yy398; yygotominor.yy398.rowPerFileBlock = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 58: /* db_optr ::= db_optr ablocks */ -{ yygotominor.yy478 = yymsp[-1].minor.yy478; yygotominor.yy478.numOfAvgCacheBlocks = strtod(yymsp[0].minor.yy0.z, NULL); } + case 85: /* db_optr ::= db_optr ablocks */ +{ yygotominor.yy398 = yymsp[-1].minor.yy398; yygotominor.yy398.numOfAvgCacheBlocks = strtod(yymsp[0].minor.yy0.z, NULL); } break; - case 59: /* db_optr ::= db_optr tblocks */ -{ yygotominor.yy478 = yymsp[-1].minor.yy478; yygotominor.yy478.numOfBlocksPerTable = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 86: /* db_optr ::= db_optr tblocks */ +{ yygotominor.yy398 = yymsp[-1].minor.yy398; yygotominor.yy398.numOfBlocksPerTable = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 60: /* db_optr ::= db_optr ctime */ -{ yygotominor.yy478 = yymsp[-1].minor.yy478; yygotominor.yy478.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 87: /* db_optr ::= db_optr ctime */ +{ yygotominor.yy398 = yymsp[-1].minor.yy398; yygotominor.yy398.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 61: /* db_optr ::= db_optr clog */ -{ yygotominor.yy478 = yymsp[-1].minor.yy478; yygotominor.yy478.commitLog = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 88: /* db_optr ::= db_optr clog */ +{ yygotominor.yy398 = yymsp[-1].minor.yy398; yygotominor.yy398.commitLog = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 62: /* db_optr ::= db_optr comp */ -{ yygotominor.yy478 = yymsp[-1].minor.yy478; yygotominor.yy478.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 89: /* db_optr ::= db_optr comp */ +{ yygotominor.yy398 = yymsp[-1].minor.yy398; yygotominor.yy398.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 63: /* db_optr ::= db_optr prec */ -{ yygotominor.yy478 = yymsp[-1].minor.yy478; yygotominor.yy478.precision = yymsp[0].minor.yy0; } + case 90: /* db_optr ::= db_optr prec */ +{ yygotominor.yy398 = yymsp[-1].minor.yy398; yygotominor.yy398.precision = yymsp[0].minor.yy0; } break; - case 64: /* db_optr ::= db_optr keep */ -{ yygotominor.yy478 = yymsp[-1].minor.yy478; yygotominor.yy478.keep = yymsp[0].minor.yy216; } + case 91: /* db_optr ::= db_optr keep */ +{ yygotominor.yy398 = yymsp[-1].minor.yy398; yygotominor.yy398.keep = yymsp[0].minor.yy56; } break; - case 65: /* alter_db_optr ::= REPLICA tagitem */ -{ - yygotominor.yy478.replica = yymsp[0].minor.yy266.i64Key; -} + case 92: /* alter_db_optr ::= */ +{ memset(&yygotominor.yy398, 0, sizeof(SCreateDBInfo));} break; - case 66: /* typename ::= ids */ -{ tSQLSetColumnType (&yygotominor.yy343, &yymsp[0].minor.yy0); } + case 95: /* typename ::= ids */ +{ tSQLSetColumnType (&yygotominor.yy223, &yymsp[0].minor.yy0); } break; - case 67: /* typename ::= ids LP signed RP */ + case 96: /* typename ::= ids LP signed RP */ { - yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy189; // negative value of name length - tSQLSetColumnType(&yygotominor.yy343, &yymsp[-3].minor.yy0); + yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy389; // negative value of name length + tSQLSetColumnType(&yygotominor.yy223, &yymsp[-3].minor.yy0); } break; - case 68: /* signed ::= INTEGER */ - case 69: /* signed ::= PLUS INTEGER */ yytestcase(yyruleno==69); -{ yygotominor.yy189 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 97: /* signed ::= INTEGER */ + case 98: /* signed ::= PLUS INTEGER */ yytestcase(yyruleno==98); +{ yygotominor.yy389 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 70: /* signed ::= MINUS INTEGER */ -{ yygotominor.yy189 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} + case 99: /* signed ::= MINUS INTEGER */ +{ yygotominor.yy389 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} break; - case 71: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ + case 100: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ { yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; setCreatedMeterName(pInfo, &yymsp[-2].minor.yy0, &yymsp[-3].minor.yy0); } break; - case 72: /* create_table_args ::= LP columnlist RP */ + case 101: /* create_table_args ::= LP columnlist RP */ { - yygotominor.yy278 = tSetCreateSQLElems(yymsp[-1].minor.yy151, NULL, NULL, NULL, NULL, TSQL_CREATE_NORMAL_METER); - setSQLInfo(pInfo, yygotominor.yy278, NULL, TSQL_CREATE_NORMAL_METER); + yygotominor.yy158 = tSetCreateSQLElems(yymsp[-1].minor.yy471, NULL, NULL, NULL, NULL, TSQL_CREATE_NORMAL_METER); + setSQLInfo(pInfo, yygotominor.yy158, NULL, TSQL_CREATE_NORMAL_METER); } break; - case 73: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ + case 102: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ { - yygotominor.yy278 = tSetCreateSQLElems(yymsp[-5].minor.yy151, yymsp[-1].minor.yy151, NULL, NULL, NULL, TSQL_CREATE_NORMAL_METRIC); - setSQLInfo(pInfo, yygotominor.yy278, NULL, TSQL_CREATE_NORMAL_METRIC); + yygotominor.yy158 = tSetCreateSQLElems(yymsp[-5].minor.yy471, yymsp[-1].minor.yy471, NULL, NULL, NULL, TSQL_CREATE_NORMAL_METRIC); + setSQLInfo(pInfo, yygotominor.yy158, NULL, TSQL_CREATE_NORMAL_METRIC); } break; - case 74: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ + case 103: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; - yygotominor.yy278 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy216, NULL, TSQL_CREATE_METER_FROM_METRIC); - setSQLInfo(pInfo, yygotominor.yy278, NULL, TSQL_CREATE_METER_FROM_METRIC); + yygotominor.yy158 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy56, NULL, TSQL_CREATE_METER_FROM_METRIC); + setSQLInfo(pInfo, yygotominor.yy158, NULL, TSQL_CREATE_METER_FROM_METRIC); } break; - case 75: /* create_table_args ::= AS select */ + case 104: /* create_table_args ::= AS select */ { - yygotominor.yy278 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy24, TSQL_CREATE_STREAM); - setSQLInfo(pInfo, yygotominor.yy278, NULL, TSQL_CREATE_STREAM); + yygotominor.yy158 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy24, TSQL_CREATE_STREAM); + setSQLInfo(pInfo, yygotominor.yy158, NULL, TSQL_CREATE_STREAM); } break; - case 76: /* columnlist ::= columnlist COMMA column */ -{yygotominor.yy151 = tFieldListAppend(yymsp[-2].minor.yy151, &yymsp[0].minor.yy343); } + case 105: /* columnlist ::= columnlist COMMA column */ +{yygotominor.yy471 = tFieldListAppend(yymsp[-2].minor.yy471, &yymsp[0].minor.yy223); } break; - case 77: /* columnlist ::= column */ -{yygotominor.yy151 = tFieldListAppend(NULL, &yymsp[0].minor.yy343);} + case 106: /* columnlist ::= column */ +{yygotominor.yy471 = tFieldListAppend(NULL, &yymsp[0].minor.yy223);} break; - case 78: /* column ::= ids typename */ + case 107: /* column ::= ids typename */ { - tSQLSetColumnInfo(&yygotominor.yy343, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy343); + tSQLSetColumnInfo(&yygotominor.yy223, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy223); } break; - case 79: /* tagitemlist ::= tagitemlist COMMA tagitem */ -{ yygotominor.yy216 = tVariantListAppend(yymsp[-2].minor.yy216, &yymsp[0].minor.yy266, -1); } + case 108: /* tagitemlist ::= tagitemlist COMMA tagitem */ +{ yygotominor.yy56 = tVariantListAppend(yymsp[-2].minor.yy56, &yymsp[0].minor.yy186, -1); } break; - case 80: /* tagitemlist ::= tagitem */ -{ yygotominor.yy216 = tVariantListAppend(NULL, &yymsp[0].minor.yy266, -1); } + case 109: /* tagitemlist ::= tagitem */ +{ yygotominor.yy56 = tVariantListAppend(NULL, &yymsp[0].minor.yy186, -1); } break; - case 81: /* tagitem ::= INTEGER */ - case 82: /* tagitem ::= FLOAT */ yytestcase(yyruleno==82); - case 83: /* tagitem ::= STRING */ yytestcase(yyruleno==83); - case 84: /* tagitem ::= BOOL */ yytestcase(yyruleno==84); -{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yygotominor.yy266, &yymsp[0].minor.yy0); } + case 110: /* tagitem ::= INTEGER */ + case 111: /* tagitem ::= FLOAT */ yytestcase(yyruleno==111); + case 112: /* tagitem ::= STRING */ yytestcase(yyruleno==112); + case 113: /* tagitem ::= BOOL */ yytestcase(yyruleno==113); +{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yygotominor.yy186, &yymsp[0].minor.yy0); } break; - case 85: /* tagitem ::= NULL */ -{ yymsp[0].minor.yy0.type = TK_STRING; toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yygotominor.yy266, &yymsp[0].minor.yy0); } + case 114: /* tagitem ::= NULL */ +{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yygotominor.yy186, &yymsp[0].minor.yy0); } break; - case 86: /* tagitem ::= MINUS INTEGER */ - case 87: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==87); + case 115: /* tagitem ::= MINUS INTEGER */ + case 116: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==116); + case 117: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==117); + case 118: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==118); { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type; toTSDBType(yymsp[-1].minor.yy0.type); - tVariantCreate(&yygotominor.yy266, &yymsp[-1].minor.yy0); + tVariantCreate(&yygotominor.yy186, &yymsp[-1].minor.yy0); } break; - case 88: /* cmd ::= select */ + case 119: /* cmd ::= select */ { setSQLInfo(pInfo, yymsp[0].minor.yy24, NULL, TSQL_QUERY_METER); } break; - case 89: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + case 120: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ { - yygotominor.yy24 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy98, &yymsp[-9].minor.yy0, yymsp[-8].minor.yy370, yymsp[-4].minor.yy216, yymsp[-3].minor.yy216, &yymsp[-7].minor.yy0, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy216, &yymsp[0].minor.yy294, &yymsp[-1].minor.yy294); + yygotominor.yy24 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy498, yymsp[-9].minor.yy56, yymsp[-8].minor.yy90, yymsp[-4].minor.yy56, yymsp[-3].minor.yy56, &yymsp[-7].minor.yy0, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy56, &yymsp[0].minor.yy294, &yymsp[-1].minor.yy294); } break; - case 90: /* sclp ::= selcollist COMMA */ -{yygotominor.yy98 = yymsp[-1].minor.yy98;} + case 121: /* sclp ::= selcollist COMMA */ +{yygotominor.yy498 = yymsp[-1].minor.yy498;} break; - case 91: /* sclp ::= */ -{yygotominor.yy98 = 0;} + case 122: /* sclp ::= */ +{yygotominor.yy498 = 0;} break; - case 92: /* selcollist ::= sclp expr as */ + case 123: /* selcollist ::= sclp expr as */ { - yygotominor.yy98 = tSQLExprListAppend(yymsp[-2].minor.yy98, yymsp[-1].minor.yy370, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); + yygotominor.yy498 = tSQLExprListAppend(yymsp[-2].minor.yy498, yymsp[-1].minor.yy90, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); } break; - case 93: /* selcollist ::= sclp STAR */ + case 124: /* selcollist ::= sclp STAR */ { tSQLExpr *pNode = tSQLExprIdValueCreate(NULL, TK_ALL); - yygotominor.yy98 = tSQLExprListAppend(yymsp[-1].minor.yy98, pNode, 0); + yygotominor.yy498 = tSQLExprListAppend(yymsp[-1].minor.yy498, pNode, 0); } break; - case 94: /* as ::= AS ids */ - case 95: /* as ::= ids */ yytestcase(yyruleno==95); + case 125: /* as ::= AS ids */ + case 126: /* as ::= ids */ yytestcase(yyruleno==126); { yygotominor.yy0 = yymsp[0].minor.yy0; } break; - case 96: /* as ::= */ + case 127: /* as ::= */ { yygotominor.yy0.n = 0; } break; - case 97: /* from ::= FROM ids cpxName */ -{yygotominor.yy0 = yymsp[-1].minor.yy0; yygotominor.yy0.n += yymsp[0].minor.yy0.n;} + case 128: /* from ::= FROM tablelist */ + case 140: /* orderby_opt ::= ORDER BY sortlist */ yytestcase(yyruleno==140); + case 148: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==148); +{yygotominor.yy56 = yymsp[0].minor.yy56;} break; - case 98: /* tmvar ::= VARIABLE */ + case 129: /* tablelist ::= ids cpxName */ +{ toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yygotominor.yy56 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);} + break; + case 130: /* tablelist ::= tablelist COMMA ids cpxName */ +{ toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yygotominor.yy56 = tVariantListAppendToken(yymsp[-3].minor.yy56, &yymsp[-1].minor.yy0, -1); } + break; + case 131: /* tmvar ::= VARIABLE */ {yygotominor.yy0 = yymsp[0].minor.yy0;} break; - case 99: /* interval_opt ::= INTERVAL LP tmvar RP */ - case 104: /* sliding_opt ::= SLIDING LP tmvar RP */ yytestcase(yyruleno==104); + case 132: /* interval_opt ::= INTERVAL LP tmvar RP */ + case 137: /* sliding_opt ::= SLIDING LP tmvar RP */ yytestcase(yyruleno==137); {yygotominor.yy0 = yymsp[-1].minor.yy0; } break; - case 100: /* interval_opt ::= */ - case 105: /* sliding_opt ::= */ yytestcase(yyruleno==105); -{yygotominor.yy0.n = 0; } + case 134: /* fill_opt ::= */ +{yygotominor.yy56 = 0; } break; - case 101: /* fill_opt ::= */ -{yygotominor.yy216 = 0; } - break; - case 102: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + case 135: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ { tVariant A = {0}; toTSDBType(yymsp[-3].minor.yy0.type); tVariantCreate(&A, &yymsp[-3].minor.yy0); - tVariantListInsert(yymsp[-1].minor.yy216, &A, -1, 0); - yygotominor.yy216 = yymsp[-1].minor.yy216; + tVariantListInsert(yymsp[-1].minor.yy56, &A, -1, 0); + yygotominor.yy56 = yymsp[-1].minor.yy56; } break; - case 103: /* fill_opt ::= FILL LP ID RP */ + case 136: /* fill_opt ::= FILL LP ID RP */ { toTSDBType(yymsp[-1].minor.yy0.type); - yygotominor.yy216 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yygotominor.yy56 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); } break; - case 106: /* orderby_opt ::= */ - case 114: /* groupby_opt ::= */ yytestcase(yyruleno==114); -{yygotominor.yy216 = 0;} - break; - case 107: /* orderby_opt ::= ORDER BY sortlist */ - case 115: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==115); -{yygotominor.yy216 = yymsp[0].minor.yy216;} + case 139: /* orderby_opt ::= */ + case 147: /* groupby_opt ::= */ yytestcase(yyruleno==147); +{yygotominor.yy56 = 0;} break; - case 108: /* sortlist ::= sortlist COMMA item sortorder */ + case 141: /* sortlist ::= sortlist COMMA item sortorder */ { - yygotominor.yy216 = tVariantListAppend(yymsp[-3].minor.yy216, &yymsp[-1].minor.yy266, yymsp[0].minor.yy412); + yygotominor.yy56 = tVariantListAppend(yymsp[-3].minor.yy56, &yymsp[-1].minor.yy186, yymsp[0].minor.yy332); } break; - case 109: /* sortlist ::= item sortorder */ + case 142: /* sortlist ::= item sortorder */ { - yygotominor.yy216 = tVariantListAppend(NULL, &yymsp[-1].minor.yy266, yymsp[0].minor.yy412); + yygotominor.yy56 = tVariantListAppend(NULL, &yymsp[-1].minor.yy186, yymsp[0].minor.yy332); } break; - case 110: /* item ::= ids cpxName */ + case 143: /* item ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - tVariantCreate(&yygotominor.yy266, &yymsp[-1].minor.yy0); + tVariantCreate(&yygotominor.yy186, &yymsp[-1].minor.yy0); } break; - case 111: /* sortorder ::= ASC */ -{yygotominor.yy412 = TSQL_SO_ASC; } + case 144: /* sortorder ::= ASC */ +{yygotominor.yy332 = TSQL_SO_ASC; } break; - case 112: /* sortorder ::= DESC */ -{yygotominor.yy412 = TSQL_SO_DESC;} + case 145: /* sortorder ::= DESC */ +{yygotominor.yy332 = TSQL_SO_DESC;} break; - case 113: /* sortorder ::= */ -{yygotominor.yy412 = TSQL_SO_ASC;} + case 146: /* sortorder ::= */ +{yygotominor.yy332 = TSQL_SO_ASC;} break; - case 116: /* grouplist ::= grouplist COMMA item */ + case 149: /* grouplist ::= grouplist COMMA item */ { - yygotominor.yy216 = tVariantListAppend(yymsp[-2].minor.yy216, &yymsp[0].minor.yy266, -1); + yygotominor.yy56 = tVariantListAppend(yymsp[-2].minor.yy56, &yymsp[0].minor.yy186, -1); } break; - case 117: /* grouplist ::= item */ + case 150: /* grouplist ::= item */ { - yygotominor.yy216 = tVariantListAppend(NULL, &yymsp[0].minor.yy266, -1); + yygotominor.yy56 = tVariantListAppend(NULL, &yymsp[0].minor.yy186, -1); } break; - case 118: /* having_opt ::= */ - case 128: /* where_opt ::= */ yytestcase(yyruleno==128); - case 164: /* expritem ::= */ yytestcase(yyruleno==164); -{yygotominor.yy370 = 0;} + case 151: /* having_opt ::= */ + case 161: /* where_opt ::= */ yytestcase(yyruleno==161); + case 197: /* expritem ::= */ yytestcase(yyruleno==197); +{yygotominor.yy90 = 0;} break; - case 119: /* having_opt ::= HAVING expr */ - case 129: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==129); - case 163: /* expritem ::= expr */ yytestcase(yyruleno==163); -{yygotominor.yy370 = yymsp[0].minor.yy370;} + case 152: /* having_opt ::= HAVING expr */ + case 162: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==162); + case 196: /* expritem ::= expr */ yytestcase(yyruleno==196); +{yygotominor.yy90 = yymsp[0].minor.yy90;} break; - case 120: /* limit_opt ::= */ - case 124: /* slimit_opt ::= */ yytestcase(yyruleno==124); + case 153: /* limit_opt ::= */ + case 157: /* slimit_opt ::= */ yytestcase(yyruleno==157); {yygotominor.yy294.limit = -1; yygotominor.yy294.offset = 0;} break; - case 121: /* limit_opt ::= LIMIT signed */ - case 125: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==125); -{yygotominor.yy294.limit = yymsp[0].minor.yy189; yygotominor.yy294.offset = 0;} + case 154: /* limit_opt ::= LIMIT signed */ + case 158: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==158); +{yygotominor.yy294.limit = yymsp[0].minor.yy389; yygotominor.yy294.offset = 0;} break; - case 122: /* limit_opt ::= LIMIT signed OFFSET signed */ - case 126: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==126); -{yygotominor.yy294.limit = yymsp[-2].minor.yy189; yygotominor.yy294.offset = yymsp[0].minor.yy189;} + case 155: /* limit_opt ::= LIMIT signed OFFSET signed */ + case 159: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==159); +{yygotominor.yy294.limit = yymsp[-2].minor.yy389; yygotominor.yy294.offset = yymsp[0].minor.yy389;} break; - case 123: /* limit_opt ::= LIMIT signed COMMA signed */ - case 127: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==127); -{yygotominor.yy294.limit = yymsp[0].minor.yy189; yygotominor.yy294.offset = yymsp[-2].minor.yy189;} + case 156: /* limit_opt ::= LIMIT signed COMMA signed */ + case 160: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==160); +{yygotominor.yy294.limit = yymsp[0].minor.yy389; yygotominor.yy294.offset = yymsp[-2].minor.yy389;} break; - case 130: /* expr ::= LP expr RP */ -{yygotominor.yy370 = yymsp[-1].minor.yy370; } + case 163: /* expr ::= LP expr RP */ +{yygotominor.yy90 = yymsp[-1].minor.yy90; } break; - case 131: /* expr ::= ID */ -{yygotominor.yy370 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} + case 164: /* expr ::= ID */ +{yygotominor.yy90 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} break; - case 132: /* expr ::= ID DOT ID */ -{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yygotominor.yy370 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} + case 165: /* expr ::= ID DOT ID */ +{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yygotominor.yy90 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} break; - case 133: /* expr ::= ID DOT STAR */ -{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yygotominor.yy370 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} + case 166: /* expr ::= ID DOT STAR */ +{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yygotominor.yy90 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} break; - case 134: /* expr ::= INTEGER */ -{yygotominor.yy370 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} + case 167: /* expr ::= INTEGER */ +{yygotominor.yy90 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} break; - case 135: /* expr ::= MINUS INTEGER */ - case 136: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==136); -{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yygotominor.yy370 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} + case 168: /* expr ::= MINUS INTEGER */ + case 169: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==169); +{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yygotominor.yy90 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} break; - case 137: /* expr ::= FLOAT */ -{yygotominor.yy370 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} + case 170: /* expr ::= FLOAT */ +{yygotominor.yy90 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} break; - case 138: /* expr ::= MINUS FLOAT */ - case 139: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==139); -{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yygotominor.yy370 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} + case 171: /* expr ::= MINUS FLOAT */ + case 172: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==172); +{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yygotominor.yy90 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} break; - case 140: /* expr ::= STRING */ -{yygotominor.yy370 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} + case 173: /* expr ::= STRING */ +{yygotominor.yy90 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} break; - case 141: /* expr ::= NOW */ -{yygotominor.yy370 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } + case 174: /* expr ::= NOW */ +{yygotominor.yy90 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } break; - case 142: /* expr ::= VARIABLE */ -{yygotominor.yy370 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} + case 175: /* expr ::= VARIABLE */ +{yygotominor.yy90 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} break; - case 143: /* expr ::= BOOL */ -{yygotominor.yy370 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} + case 176: /* expr ::= BOOL */ +{yygotominor.yy90 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} break; - case 144: /* expr ::= ID LP exprlist RP */ + case 177: /* expr ::= ID LP exprlist RP */ { - yygotominor.yy370 = tSQLExprCreateFunction(yymsp[-1].minor.yy98, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); + yygotominor.yy90 = tSQLExprCreateFunction(yymsp[-1].minor.yy498, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } break; - case 145: /* expr ::= ID LP STAR RP */ + case 178: /* expr ::= ID LP STAR RP */ { - yygotominor.yy370 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); + yygotominor.yy90 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } break; - case 146: /* expr ::= expr AND expr */ -{yygotominor.yy370 = tSQLExprCreate(yymsp[-2].minor.yy370, yymsp[0].minor.yy370, TK_AND);} + case 179: /* expr ::= expr AND expr */ +{yygotominor.yy90 = tSQLExprCreate(yymsp[-2].minor.yy90, yymsp[0].minor.yy90, TK_AND);} break; - case 147: /* expr ::= expr OR expr */ -{yygotominor.yy370 = tSQLExprCreate(yymsp[-2].minor.yy370, yymsp[0].minor.yy370, TK_OR); } + case 180: /* expr ::= expr OR expr */ +{yygotominor.yy90 = tSQLExprCreate(yymsp[-2].minor.yy90, yymsp[0].minor.yy90, TK_OR); } break; - case 148: /* expr ::= expr LT expr */ -{yygotominor.yy370 = tSQLExprCreate(yymsp[-2].minor.yy370, yymsp[0].minor.yy370, TK_LT);} + case 181: /* expr ::= expr LT expr */ +{yygotominor.yy90 = tSQLExprCreate(yymsp[-2].minor.yy90, yymsp[0].minor.yy90, TK_LT);} break; - case 149: /* expr ::= expr GT expr */ -{yygotominor.yy370 = tSQLExprCreate(yymsp[-2].minor.yy370, yymsp[0].minor.yy370, TK_GT);} + case 182: /* expr ::= expr GT expr */ +{yygotominor.yy90 = tSQLExprCreate(yymsp[-2].minor.yy90, yymsp[0].minor.yy90, TK_GT);} break; - case 150: /* expr ::= expr LE expr */ -{yygotominor.yy370 = tSQLExprCreate(yymsp[-2].minor.yy370, yymsp[0].minor.yy370, TK_LE);} + case 183: /* expr ::= expr LE expr */ +{yygotominor.yy90 = tSQLExprCreate(yymsp[-2].minor.yy90, yymsp[0].minor.yy90, TK_LE);} break; - case 151: /* expr ::= expr GE expr */ -{yygotominor.yy370 = tSQLExprCreate(yymsp[-2].minor.yy370, yymsp[0].minor.yy370, TK_GE);} + case 184: /* expr ::= expr GE expr */ +{yygotominor.yy90 = tSQLExprCreate(yymsp[-2].minor.yy90, yymsp[0].minor.yy90, TK_GE);} break; - case 152: /* expr ::= expr NE expr */ -{yygotominor.yy370 = tSQLExprCreate(yymsp[-2].minor.yy370, yymsp[0].minor.yy370, TK_NE);} + case 185: /* expr ::= expr NE expr */ +{yygotominor.yy90 = tSQLExprCreate(yymsp[-2].minor.yy90, yymsp[0].minor.yy90, TK_NE);} break; - case 153: /* expr ::= expr EQ expr */ -{yygotominor.yy370 = tSQLExprCreate(yymsp[-2].minor.yy370, yymsp[0].minor.yy370, TK_EQ);} + case 186: /* expr ::= expr EQ expr */ +{yygotominor.yy90 = tSQLExprCreate(yymsp[-2].minor.yy90, yymsp[0].minor.yy90, TK_EQ);} break; - case 154: /* expr ::= expr PLUS expr */ -{yygotominor.yy370 = tSQLExprCreate(yymsp[-2].minor.yy370, yymsp[0].minor.yy370, TK_PLUS); } + case 187: /* expr ::= expr PLUS expr */ +{yygotominor.yy90 = tSQLExprCreate(yymsp[-2].minor.yy90, yymsp[0].minor.yy90, TK_PLUS); } break; - case 155: /* expr ::= expr MINUS expr */ -{yygotominor.yy370 = tSQLExprCreate(yymsp[-2].minor.yy370, yymsp[0].minor.yy370, TK_MINUS); } + case 188: /* expr ::= expr MINUS expr */ +{yygotominor.yy90 = tSQLExprCreate(yymsp[-2].minor.yy90, yymsp[0].minor.yy90, TK_MINUS); } break; - case 156: /* expr ::= expr STAR expr */ -{yygotominor.yy370 = tSQLExprCreate(yymsp[-2].minor.yy370, yymsp[0].minor.yy370, TK_STAR); } + case 189: /* expr ::= expr STAR expr */ +{yygotominor.yy90 = tSQLExprCreate(yymsp[-2].minor.yy90, yymsp[0].minor.yy90, TK_STAR); } break; - case 157: /* expr ::= expr SLASH expr */ -{yygotominor.yy370 = tSQLExprCreate(yymsp[-2].minor.yy370, yymsp[0].minor.yy370, TK_DIVIDE);} + case 190: /* expr ::= expr SLASH expr */ +{yygotominor.yy90 = tSQLExprCreate(yymsp[-2].minor.yy90, yymsp[0].minor.yy90, TK_DIVIDE);} break; - case 158: /* expr ::= expr REM expr */ -{yygotominor.yy370 = tSQLExprCreate(yymsp[-2].minor.yy370, yymsp[0].minor.yy370, TK_REM); } + case 191: /* expr ::= expr REM expr */ +{yygotominor.yy90 = tSQLExprCreate(yymsp[-2].minor.yy90, yymsp[0].minor.yy90, TK_REM); } break; - case 159: /* expr ::= expr LIKE expr */ -{yygotominor.yy370 = tSQLExprCreate(yymsp[-2].minor.yy370, yymsp[0].minor.yy370, TK_LIKE); } + case 192: /* expr ::= expr LIKE expr */ +{yygotominor.yy90 = tSQLExprCreate(yymsp[-2].minor.yy90, yymsp[0].minor.yy90, TK_LIKE); } break; - case 160: /* expr ::= expr IN LP exprlist RP */ -{yygotominor.yy370 = tSQLExprCreate(yymsp[-4].minor.yy370, (tSQLExpr*)yymsp[-1].minor.yy98, TK_IN); } + case 193: /* expr ::= expr IN LP exprlist RP */ +{yygotominor.yy90 = tSQLExprCreate(yymsp[-4].minor.yy90, (tSQLExpr*)yymsp[-1].minor.yy498, TK_IN); } break; - case 161: /* exprlist ::= exprlist COMMA expritem */ - case 168: /* itemlist ::= itemlist COMMA expr */ yytestcase(yyruleno==168); -{yygotominor.yy98 = tSQLExprListAppend(yymsp[-2].minor.yy98,yymsp[0].minor.yy370,0);} + case 194: /* exprlist ::= exprlist COMMA expritem */ + case 201: /* itemlist ::= itemlist COMMA expr */ yytestcase(yyruleno==201); +{yygotominor.yy498 = tSQLExprListAppend(yymsp[-2].minor.yy498,yymsp[0].minor.yy90,0);} break; - case 162: /* exprlist ::= expritem */ - case 169: /* itemlist ::= expr */ yytestcase(yyruleno==169); -{yygotominor.yy98 = tSQLExprListAppend(0,yymsp[0].minor.yy370,0);} + case 195: /* exprlist ::= expritem */ + case 202: /* itemlist ::= expr */ yytestcase(yyruleno==202); +{yygotominor.yy498 = tSQLExprListAppend(0,yymsp[0].minor.yy90,0);} break; - case 165: /* cmd ::= INSERT INTO cpxName insert_value_list */ + case 198: /* cmd ::= INSERT INTO cpxName insert_value_list */ { - tSetInsertSQLElems(pInfo, &yymsp[-1].minor.yy0, yymsp[0].minor.yy434); + tSetInsertSQLElems(pInfo, &yymsp[-1].minor.yy0, yymsp[0].minor.yy74); } break; - case 166: /* insert_value_list ::= VALUES LP itemlist RP */ -{yygotominor.yy434 = tSQLListListAppend(NULL, yymsp[-1].minor.yy98);} + case 199: /* insert_value_list ::= VALUES LP itemlist RP */ +{yygotominor.yy74 = tSQLListListAppend(NULL, yymsp[-1].minor.yy498);} break; - case 167: /* insert_value_list ::= insert_value_list VALUES LP itemlist RP */ -{yygotominor.yy434 = tSQLListListAppend(yymsp[-4].minor.yy434, yymsp[-1].minor.yy98);} + case 200: /* insert_value_list ::= insert_value_list VALUES LP itemlist RP */ +{yygotominor.yy74 = tSQLListListAppend(yymsp[-4].minor.yy74, yymsp[-1].minor.yy498);} break; - case 170: /* cmd ::= RESET QUERY CACHE */ + case 203: /* cmd ::= RESET QUERY CACHE */ { setDCLSQLElems(pInfo, RESET_QUERY_CACHE, 0);} break; - case 171: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 204: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy151, NULL, ALTER_TABLE_ADD_COLUMN); + SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy471, NULL, ALTER_TABLE_ADD_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, ALTER_TABLE_ADD_COLUMN); } break; - case 172: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 205: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -2037,14 +2201,14 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, ALTER_TABLE_DROP_COLUMN); } break; - case 173: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 206: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy151, NULL, ALTER_TABLE_TAGS_ADD); + SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy471, NULL, ALTER_TABLE_TAGS_ADD); setSQLInfo(pInfo, pAlterTable, NULL, ALTER_TABLE_TAGS_ADD); } break; - case 174: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 207: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -2055,7 +2219,7 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, ALTER_TABLE_TAGS_DROP); } break; - case 175: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 208: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -2069,25 +2233,25 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, ALTER_TABLE_TAGS_CHG); } break; - case 176: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 209: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; toTSDBType(yymsp[-2].minor.yy0.type); tVariantList* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tVariantListAppend(A, &yymsp[0].minor.yy266, -1); + A = tVariantListAppend(A, &yymsp[0].minor.yy186, -1); SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-6].minor.yy0, NULL, A, ALTER_TABLE_TAGS_SET); setSQLInfo(pInfo, pAlterTable, NULL, ALTER_TABLE_TAGS_SET); } break; - case 177: /* cmd ::= KILL CONNECTION IP COLON INTEGER */ + case 210: /* cmd ::= KILL CONNECTION IP COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setDCLSQLElems(pInfo, KILL_CONNECTION, 1, &yymsp[-2].minor.yy0);} break; - case 178: /* cmd ::= KILL STREAM IP COLON INTEGER COLON INTEGER */ + case 211: /* cmd ::= KILL STREAM IP COLON INTEGER COLON INTEGER */ {yymsp[-4].minor.yy0.n += (yymsp[-3].minor.yy0.n + yymsp[-2].minor.yy0.n + yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setDCLSQLElems(pInfo, KILL_STREAM, 1, &yymsp[-4].minor.yy0);} break; - case 179: /* cmd ::= KILL QUERY IP COLON INTEGER COLON INTEGER */ + case 212: /* cmd ::= KILL QUERY IP COLON INTEGER COLON INTEGER */ {yymsp[-4].minor.yy0.n += (yymsp[-3].minor.yy0.n + yymsp[-2].minor.yy0.n + yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setDCLSQLElems(pInfo, KILL_QUERY, 1, &yymsp[-4].minor.yy0);} break; default: diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 52d8638c09..7804cb0d0e 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -593,11 +593,13 @@ void *taosAddDataIntoCache(void *handle, char *key, char *pData, int dataSize, i taosHashTableResize(pObj); pNode = taosAddToCacheImpl(pObj, key, keyLen, pData, dataSize, keepTime * 1000L); - pTrace( - "key:%s %p added into cache, slot:%d, addTime:%lld, expireTime:%lld, cache total:%d, " - "size:%lldbytes, collision:%d", - pNode->key, pNode, HASH_INDEX(pNode->hashVal, pObj->capacity), pNode->addTime, pNode->time, pObj->size, - pObj->totalSize, pObj->statistics.numOfCollision); + if (NULL != pNode) { + pTrace( + "key:%s %p added into cache, slot:%d, addTime:%lld, expireTime:%lld, cache total:%d, " + "size:%lldbytes, collision:%d", + pNode->key, pNode, HASH_INDEX(pNode->hashVal, pObj->capacity), pNode->addTime, pNode->time, pObj->size, + pObj->totalSize, pObj->statistics.numOfCollision); + } } else { // old data exists, update the node pNode = taosUpdateCacheImpl(pObj, pOldNode, key, keyLen, pData, dataSize, keepTime * 1000L); pTrace("key:%s %p exist in cache, updated", key, pNode); diff --git a/src/util/src/tcrc32c.c b/src/util/src/tcrc32c.c index 5c484994a5..88841bcbc9 100644 --- a/src/util/src/tcrc32c.c +++ b/src/util/src/tcrc32c.c @@ -17,7 +17,10 @@ misrepresented as being the original software. 3. This notice may not be removed or altered from any source distribution. */ +#ifndef _TD_ARM_ #include +#endif + #include #include @@ -1186,11 +1189,16 @@ uint32_t crc32c_sf(uint32_t crci, crc_stream input, size_t length) { } /* Apply the zeros operator table to crc. */ -static inline uint32_t shift_crc(uint32_t shift_table[][256], uint32_t crc) { +static uint32_t shift_crc(uint32_t shift_table[][256], uint32_t crc) { return shift_table[0][crc & 0xff] ^ shift_table[1][(crc >> 8) & 0xff] ^ shift_table[2][(crc >> 16) & 0xff] ^ shift_table[3][crc >> 24]; } +/* Compute a CRC-32C. If the crc32 instruction is available, use the hardware + version. Otherwise, use the software version. */ +uint32_t (*crc32c)(uint32_t crci, crc_stream bytes, size_t len) = crc32c_sf; + +#ifndef _TD_ARM_ /* Compute CRC-32C using the Intel hardware instruction. */ uint32_t crc32c_hw(uint32_t crc, crc_stream buf, size_t len) { crc_stream next = buf; @@ -1342,18 +1350,21 @@ uint32_t crc32c_hw(uint32_t crc, crc_stream buf, size_t len) { (have) = (ecx >> 20) & 1; \ } while (0) -/* Compute a CRC-32C. If the crc32 instruction is available, use the hardware - version. Otherwise, use the software version. */ -uint32_t (*crc32c)(uint32_t crci, crc_stream bytes, size_t len) = NULL; +#endif // #ifndef _TD_ARM_ void taosResolveCRC() { +#ifndef _TD_ARM_32 int sse42; SSE42(sse42); crc32c = sse42 ? crc32c_hw : crc32c_sf; +#else + crc32c = crc32c_sf; +#endif /* return sse42 ? crc32c_hw(crci, bytes, len) : crc32c_sf(crci, bytes, len); */ } + #ifdef TEST_CRC32C_MAIN #include #include diff --git a/src/util/src/textbuffer.c b/src/util/src/textbuffer.c index aa0ce9e851..c6b56919e9 100644 --- a/src/util/src/textbuffer.c +++ b/src/util/src/textbuffer.c @@ -39,22 +39,28 @@ #define COLMODEL_GET_VAL(data, schema, allrow, rowId, colId) \ (data + (schema)->colOffset[colId] * (allrow) + (rowId) * (schema)->pFields[colId].bytes) -void getExtTmpfilePath(const char *fileNamePattern, int64_t serialNumber, int32_t seg, int32_t slot, char *dstPath) { - char tmpPath[512] = {0}; +int32_t tmpFileSerialNum = 0; - char *tmpDir = NULL; +void getTmpfilePath(const char *fileNamePrefix, char *dstPath) { + const char* tdengineTmpFileNamePrefix = "tdengine-"; + + char tmpPath[MAX_TMPFILE_PATH_LENGTH] = {0}; #ifdef WINDOWS - tmpDir = getenv("tmp"); + char *tmpDir = getenv("tmp"); + if (tmpDir == NULL) { + tmpDir = ""; + } #else - tmpDir = "/tmp/"; + char *tmpDir = "/tmp/"; #endif - strcat(tmpPath, tmpDir); - strcat(tmpPath, fileNamePattern); + strcpy(tmpPath, tmpDir); + strcat(tmpPath, tdengineTmpFileNamePrefix); + strcat(tmpPath, fileNamePrefix); + strcat(tmpPath, "-%u-%u"); - int32_t ret = sprintf(dstPath, tmpPath, taosGetTimestampUs(), serialNumber, seg, slot); - dstPath[ret] = 0; // ensure null-terminated string + snprintf(dstPath, MAX_TMPFILE_PATH_LENGTH, tmpPath, taosGetPthreadId(), __sync_add_and_fetch_32(&tmpFileSerialNum, 1)); } /* @@ -471,95 +477,96 @@ void tBucketDoubleHash(tMemBucket *pBucket, void *value, int16_t *segIdx, int16_ } } -void tMemBucketCreate(tMemBucket **pBucket, int32_t totalSlots, int32_t nBufferSize, int16_t nElemSize, - int16_t dataType, tOrderDescriptor *pDesc) { - *pBucket = (tMemBucket *)malloc(sizeof(tMemBucket)); +tMemBucket* tMemBucketCreate(int32_t totalSlots, int32_t nBufferSize, int16_t nElemSize, int16_t dataType, tOrderDescriptor *pDesc) { + tMemBucket* pBucket = (tMemBucket *)malloc(sizeof(tMemBucket)); - (*pBucket)->nTotalSlots = totalSlots; - (*pBucket)->nSlotsOfSeg = 1 << 6; // 64 Segments, 16 slots each seg. - (*pBucket)->dataType = dataType; - (*pBucket)->nElemSize = nElemSize; - (*pBucket)->nPageSize = DEFAULT_PAGE_SIZE; + pBucket->nTotalSlots = totalSlots; + pBucket->nSlotsOfSeg = 1 << 6; // 64 Segments, 16 slots each seg. + pBucket->dataType = dataType; + pBucket->nElemSize = nElemSize; + pBucket->nPageSize = DEFAULT_PAGE_SIZE; - (*pBucket)->numOfElems = 0; - (*pBucket)->numOfSegs = (*pBucket)->nTotalSlots / (*pBucket)->nSlotsOfSeg; + pBucket->numOfElems = 0; + pBucket->numOfSegs = pBucket->nTotalSlots / pBucket->nSlotsOfSeg; - (*pBucket)->nTotalBufferSize = nBufferSize; + pBucket->nTotalBufferSize = nBufferSize; - (*pBucket)->maxElemsCapacity = (*pBucket)->nTotalBufferSize / (*pBucket)->nElemSize; + pBucket->maxElemsCapacity = pBucket->nTotalBufferSize / pBucket->nElemSize; - (*pBucket)->numOfTotalPages = (*pBucket)->nTotalBufferSize / (*pBucket)->nPageSize; - (*pBucket)->numOfAvailPages = (*pBucket)->numOfTotalPages; + pBucket->numOfTotalPages = pBucket->nTotalBufferSize / pBucket->nPageSize; + pBucket->numOfAvailPages = pBucket->numOfTotalPages; - (*pBucket)->pOrderDesc = pDesc; + pBucket->pOrderDesc = pDesc; - switch ((*pBucket)->dataType) { + switch (pBucket->dataType) { case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_TINYINT: { - (*pBucket)->nRange.iMinVal = INT32_MAX; - (*pBucket)->nRange.iMaxVal = INT32_MIN; - (*pBucket)->HashFunc = tBucketIntHash; + pBucket->nRange.iMinVal = INT32_MAX; + pBucket->nRange.iMaxVal = INT32_MIN; + pBucket->HashFunc = tBucketIntHash; break; }; case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_FLOAT: { - (*pBucket)->nRange.dMinVal = DBL_MAX; - (*pBucket)->nRange.dMaxVal = -DBL_MAX; - (*pBucket)->HashFunc = tBucketDoubleHash; + pBucket->nRange.dMinVal = DBL_MAX; + pBucket->nRange.dMaxVal = -DBL_MAX; + pBucket->HashFunc = tBucketDoubleHash; break; }; case TSDB_DATA_TYPE_BIGINT: { - (*pBucket)->nRange.i64MinVal = INT64_MAX; - (*pBucket)->nRange.i64MaxVal = INT64_MIN; - (*pBucket)->HashFunc = tBucketBigIntHash; + pBucket->nRange.i64MinVal = INT64_MAX; + pBucket->nRange.i64MaxVal = INT64_MIN; + pBucket->HashFunc = tBucketBigIntHash; break; }; default: { - pError("MemBucket:%p,not support data type %d,failed", *pBucket, (*pBucket)->dataType); - tfree(*pBucket); - return; + pError("MemBucket:%p,not support data type %d,failed", *pBucket, pBucket->dataType); + tfree(pBucket); + return NULL; } } if (pDesc->pSchema->numOfCols != 1 || pDesc->pSchema->colOffset[0] != 0) { pError("MemBucket:%p,only consecutive data is allowed,invalid numOfCols:%d or offset:%d", *pBucket, pDesc->pSchema->numOfCols, pDesc->pSchema->colOffset[0]); - tfree(*pBucket); - return; + tfree(pBucket); + return NULL; } if (pDesc->pSchema->pFields[0].type != dataType) { pError("MemBucket:%p,data type is not consistent,%d in schema, %d in param", *pBucket, pDesc->pSchema->pFields[0].type, dataType); - tfree(*pBucket); - return; + tfree(pBucket); + return NULL; } - if ((*pBucket)->numOfTotalPages < (*pBucket)->nTotalSlots) { - pWarn("MemBucket:%p,total buffer pages %d are not enough for all slots", *pBucket, (*pBucket)->numOfTotalPages); + if (pBucket->numOfTotalPages < pBucket->nTotalSlots) { + pWarn("MemBucket:%p,total buffer pages %d are not enough for all slots", *pBucket, pBucket->numOfTotalPages); } - (*pBucket)->pSegs = (tMemBucketSegment *)malloc((*pBucket)->numOfSegs * sizeof(tMemBucketSegment)); + pBucket->pSegs = (tMemBucketSegment *)malloc(pBucket->numOfSegs * sizeof(tMemBucketSegment)); - for (int32_t i = 0; i < (*pBucket)->numOfSegs; ++i) { - (*pBucket)->pSegs[i].numOfSlots = (*pBucket)->nSlotsOfSeg; - (*pBucket)->pSegs[i].pBuffer = NULL; - (*pBucket)->pSegs[i].pBoundingEntries = NULL; + for (int32_t i = 0; i < pBucket->numOfSegs; ++i) { + pBucket->pSegs[i].numOfSlots = pBucket->nSlotsOfSeg; + pBucket->pSegs[i].pBuffer = NULL; + pBucket->pSegs[i].pBoundingEntries = NULL; } - pTrace("MemBucket:%p,created,buffer size:%d,elem size:%d", *pBucket, (*pBucket)->numOfTotalPages * DEFAULT_PAGE_SIZE, - (*pBucket)->nElemSize); + pTrace("MemBucket:%p,created,buffer size:%d,elem size:%d", *pBucket, pBucket->numOfTotalPages * DEFAULT_PAGE_SIZE, + pBucket->nElemSize); + + return pBucket; } -void tMemBucketDestroy(tMemBucket **pBucket) { - if (*pBucket == NULL) { +void tMemBucketDestroy(tMemBucket *pBucket) { + if (pBucket == NULL) { return; } - if ((*pBucket)->pSegs) { - for (int32_t i = 0; i < (*pBucket)->numOfSegs; ++i) { - tMemBucketSegment *pSeg = &((*pBucket)->pSegs[i]); + if (pBucket->pSegs) { + for (int32_t i = 0; i < pBucket->numOfSegs; ++i) { + tMemBucketSegment *pSeg = &(pBucket->pSegs[i]); tfree(pSeg->pBoundingEntries); if (pSeg->pBuffer == NULL || pSeg->numOfSlots == 0) { @@ -575,8 +582,8 @@ void tMemBucketDestroy(tMemBucket **pBucket) { } } - tfree((*pBucket)->pSegs); - tfree(*pBucket); + tfree(pBucket->pSegs); + tfree(pBucket); } /* @@ -761,9 +768,9 @@ void tMemBucketPut(tMemBucket *pBucket, void *data, int32_t numOfRows) { } if (pSeg->pBuffer[slotIdx] == NULL) { - int64_t pid = taosGetPthreadId(); - char name[512] = {0}; - getExtTmpfilePath("/tb_ex_bk_%lld_%lld_%d_%d", pid, segIdx, slotIdx, name); + char name[MAX_TMPFILE_PATH_LENGTH] = {0}; + getTmpfilePath("tb_ex_bk_%lld_%lld_%d_%d", name); + tExtMemBufferCreate(&pSeg->pBuffer[slotIdx], pBucket->numOfTotalPages * pBucket->nPageSize, pBucket->nElemSize, name, pBucket->pOrderDesc->pSchema); pSeg->pBuffer[slotIdx]->flushModel = SINGLE_APPEND_MODEL; @@ -833,6 +840,7 @@ static FORCE_INLINE int32_t primaryKeyComparator(int64_t f1, int64_t f2, int32_t } } +// todo refactor static FORCE_INLINE int32_t columnValueAscendingComparator(char *f1, char *f2, int32_t type, int32_t bytes) { switch (type) { case TSDB_DATA_TYPE_INT: { @@ -1378,31 +1386,26 @@ static void printBinaryData(char *data, int32_t len) { static void printBinaryDataEx(char *data, int32_t len, SSrcColumnInfo *param) { if (param->functionId == TSDB_FUNC_LAST_DST) { switch (param->type) { - case TSDB_DATA_TYPE_TINYINT: - printf("%lld,%d\t", *(int64_t *)data, *(int8_t *)(data + TSDB_KEYSIZE + 1)); + case TSDB_DATA_TYPE_TINYINT:printf("%lld,%d\t", *(int64_t *) data, *(int8_t *) (data + TSDB_KEYSIZE + 1)); break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%lld,%d\t", *(int64_t *)data, *(int16_t *)(data + TSDB_KEYSIZE + 1)); + case TSDB_DATA_TYPE_SMALLINT:printf("%lld,%d\t", *(int64_t *) data, *(int16_t *) (data + TSDB_KEYSIZE + 1)); break; case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - printf("%lld,%lld\t", *(int64_t *)data, *(int64_t *)(data + TSDB_KEYSIZE + 1)); + case TSDB_DATA_TYPE_BIGINT:printf("%lld,%lld\t", *(int64_t *) data, *(int64_t *) (data + TSDB_KEYSIZE + 1)); break; - case TSDB_DATA_TYPE_FLOAT: - printf("%lld,%d\t", *(int64_t *)data, *(float *)(data + TSDB_KEYSIZE + 1)); + case TSDB_DATA_TYPE_FLOAT:printf("%lld,%d\t", *(int64_t *) data, *(float *) (data + TSDB_KEYSIZE + 1)); break; - case TSDB_DATA_TYPE_DOUBLE: - printf("%lld,%d\t", *(int64_t *)data, *(double *)(data + TSDB_KEYSIZE + 1)); + case TSDB_DATA_TYPE_DOUBLE:printf("%lld,%d\t", *(int64_t *) data, *(double *) (data + TSDB_KEYSIZE + 1)); break; - case TSDB_DATA_TYPE_BINARY: - printf("%lld,%s\t", *(int64_t *)data, (data + TSDB_KEYSIZE + 1)); + case TSDB_DATA_TYPE_BINARY:printf("%lld,%s\t", *(int64_t *) data, (data + TSDB_KEYSIZE + 1)); break; case TSDB_DATA_TYPE_INT: - default: - printf("%lld,%d\t", *(int64_t *)data, *(int32_t *)(data + TSDB_KEYSIZE + 1)); + default:printf("%lld,%d\t", *(int64_t *) data, *(int32_t *) (data + TSDB_KEYSIZE + 1)); break; } + } else if (param->functionId == TSDB_FUNC_AVG) { + printf("%f,%lld\t", *(double *) data, *(int64_t *) (data + sizeof(double) + 1)); } else { // functionId == TSDB_FUNC_MAX_DST | TSDB_FUNC_TAG switch (param->type) { @@ -1428,7 +1431,7 @@ static void printBinaryDataEx(char *data, int32_t len, SSrcColumnInfo *param) { case TSDB_DATA_TYPE_INT: default: - printf("%d\t", *(int32_t *)data); + printf("%d\t", *(double *)data); break; } } @@ -1439,9 +1442,10 @@ void tColModelDisplay(tColModel *pModel, void *pData, int32_t numOfRows, int32_t for (int32_t j = 0; j < pModel->numOfCols; ++j) { char *val = COLMODEL_GET_VAL((char *)pData, pModel, totalCapacity, i, j); - printf("type:%d\t", pModel->pFields[j].type); + int type = pModel->pFields[j].type; + printf("type:%d ", type); - switch (pModel->pFields[j].type) { + switch (type) { case TSDB_DATA_TYPE_BIGINT: printf("%lld\t", *(int64_t *)val); break; @@ -1542,10 +1546,10 @@ void tColModelCompact(tColModel *pModel, tFilePage *inputBuffer, int32_t maxElem } /* start from the second column */ - for (int32_t m = 1; m < pModel->numOfCols; ++m) { - memmove(inputBuffer->data + pModel->colOffset[m] * inputBuffer->numOfElems, - inputBuffer->data + pModel->colOffset[m] * maxElemsCapacity, - pModel->pFields[m].bytes * inputBuffer->numOfElems); + for (int32_t i = 1; i < pModel->numOfCols; ++i) { + memmove(inputBuffer->data + pModel->colOffset[i] * inputBuffer->numOfElems, + inputBuffer->data + pModel->colOffset[i] * maxElemsCapacity, + pModel->pFields[i].bytes * inputBuffer->numOfElems); } } @@ -1559,11 +1563,11 @@ void tColModelErase(tColModel *pModel, tFilePage *inputBuffer, int32_t maxCapaci int32_t secPart = inputBuffer->numOfElems - e - 1; /* start from the second column */ - for (int32_t m = 0; m < pModel->numOfCols; ++m) { - char *startPos = inputBuffer->data + pModel->colOffset[m] * maxCapacity + s * pModel->pFields[m].bytes; - char *endPos = startPos + pModel->pFields[m].bytes * removed; + for (int32_t i = 0; i < pModel->numOfCols; ++i) { + char *startPos = inputBuffer->data + pModel->colOffset[i] * maxCapacity + s * pModel->pFields[i].bytes; + char *endPos = startPos + pModel->pFields[i].bytes * removed; - memmove(startPos, endPos, pModel->pFields[m].bytes * secPart); + memmove(startPos, endPos, pModel->pFields[i].bytes * secPart); } inputBuffer->numOfElems = remain; diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index 9fd065e90a..9abe60220f 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -13,14 +13,9 @@ * along with this program. If not, see . */ +#include "os.h" #include -#include -#include -#include -#include -#include -#include "os.h" #include "tglobalcfg.h" #include "tkey.h" #include "tlog.h" @@ -29,11 +24,16 @@ #include "tsystem.h" #include "tutil.h" +// monitor module api +int (*startMonitor)() = NULL; +void (*stopMonitor)() = NULL; + // system info, not configurable int64_t tsPageSize; int64_t tsOpenMax; int64_t tsStreamMax; int32_t tsNumOfCores = 1; +int32_t tsAlternativeRole = 0; float tsTotalLogDirGB = 0; float tsTotalTmpDirGB = 0; float tsTotalDataDirGB = 0; @@ -56,8 +56,14 @@ int tscEmbedded = 0; */ int64_t tsMsPerDay[] = {86400000L, 86400000000L}; +char tsMasterIp[TSDB_IPv4ADDR_LEN] = {0}; +char tsSecondIp[TSDB_IPv4ADDR_LEN] = {0}; short tsMgmtShellPort = 6030; // udp[6030-6034] tcp[6030] short tsVnodeShellPort = 6035; // udp[6035-6039] tcp[6035] +short tsMgmtVnodePort = 6040; // udp[6040-6044] tcp[6040] +short tsVnodeVnodePort = 6045; // tcp[6045] +short tsMgmtMgmtPort = 6050; // sdbPeerPort only udp, numOfVnodes fixed to 1, range udp[6050] +short tsMgmtSyncPort = 6050; // sdbSyncPort only tcp, range tcp[6050] int tsStatusInterval = 1; // second int tsShellActivityTimer = 3; // second @@ -68,7 +74,9 @@ int tsMetricMetaKeepTimer = 600; // second float tsNumOfThreadsPerCore = 1.0; float tsRatioOfQueryThreads = 0.5; +char tsPublicIp[TSDB_IPv4ADDR_LEN] = {0}; char tsInternalIp[TSDB_IPv4ADDR_LEN] = {0}; +char tsPrivateIp[TSDB_IPv4ADDR_LEN] = {0}; char tsServerIpStr[TSDB_IPv4ADDR_LEN] = "0.0.0.0"; short tsNumOfVnodesPerCore = 8; short tsNumOfTotalVnodes = 0; @@ -87,15 +95,19 @@ short tsCommitLog = 1; short tsCompression = 2; short tsDaysPerFile = 10; int tsDaysToKeep = 3650; +int tsReplications = 1; +int tsNumOfMPeers = 3; int tsMaxShellConns = 2000; +int tsMaxAccounts = 100; int tsMaxUsers = 1000; int tsMaxDbs = 1000; int tsMaxTables = 650000; int tsMaxDnodes = 1000; int tsMaxVGroups = 1000; +char tsMgmtZone[16] = "rzone"; -char tsLocalIp[TSDB_IPv4ADDR_LEN] = "0.0.0.0"; +char tsLocalIp[TSDB_IPv4ADDR_LEN] = {0}; char tsDefaultDB[TSDB_DB_NAME_LEN] = {0}; char tsDefaultUser[64] = "root"; char tsDefaultPass[64] = "taosdata"; @@ -103,10 +115,27 @@ int tsMaxMeterConnections = 10000; int tsMaxMgmtConnections = 2000; int tsMaxVnodeConnections = 10000; +int tsBalanceMonitorInterval = 2; // seconds +int tsBalanceStartInterval = 300; // seconds +int tsBalancePolicy = 0; // 1-use sys.montor +int tsOfflineThreshold = 864000; // seconds 10days +int tsMgmtEqualVnodeNum = 0; + int tsEnableHttpModule = 1; int tsEnableMonitorModule = 1; int tsRestRowLimit = 10240; +/* + * denote if the server needs to compress response message at the application layer to client, including query rsp, + * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server. + * + * 0: all data are compressed + * -1: all data are not compressed + * other values: if the message payload size is greater than the tsCompressMsgSize, the message will be compressed. + */ +int tsCompressMsgSize = -1; + +char tsSocketType[4] = "udp"; // use UDP by default[option: udp, tcp] int tsTimePrecision = TSDB_TIME_PRECISION_MILLI; // time precision, millisecond by default int tsMinSlidingTime = 10; // 10 ms for sliding time, the value will changed in // case of time precision changed @@ -122,7 +151,6 @@ int tsStreamCompRetryDelay = 10; // the stream computing delay int tsProjectExecInterval = 10000; // every 10sec, the projection will be executed once int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance -char tsSocketType[4] = "udp"; // use UDP by default[option: udp, tcp] char tsHttpIp[TSDB_IPv4ADDR_LEN] = "0.0.0.0"; short tsHttpPort = 6020; // only tcp, range tcp[6020] // short tsNginxPort = 6060; //only tcp, range tcp[6060] @@ -130,9 +158,14 @@ int tsHttpCacheSessions = 100; int tsHttpSessionExpire = 36000; int tsHttpMaxThreads = 2; int tsHttpEnableCompress = 0; +int tsHttpEnableRecordSql = 0; int tsTelegrafUseFieldNum = 0; +int tsAdminRowLimit = 10240; + +int tsRpcTimer = 300; +int tsRpcMaxTime = 600; // seconds; -char tsMonitorDbName[] = "log"; +char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log"; int tsMonitorInterval = 30; // seconds char tsInternalPass[] = "secretkey"; @@ -141,6 +174,7 @@ char tsLocale[TSDB_LOCALE_LEN] = {0}; char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string int tsNumOfLogLines = 10000000; +uint32_t rpcDebugFlag = 131; uint32_t ddebugFlag = 131; uint32_t mdebugFlag = 135; uint32_t sdbDebugFlag = 135; @@ -258,6 +292,21 @@ void tsReadFilePathConfig(SGlobalConfig *cfg, char *input_value) { } } +void tsExpandFilePath(char* option_name, char* input_value) { + wordexp_t full_path; + wordexp(input_value, &full_path, 0); + if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) { + strcpy(input_value, full_path.we_wordv[0]); + } + wordfree(&full_path); + + struct stat dirstat; + if (stat(input_value, &dirstat) < 0) { + int code = mkdir(input_value, 0755); + pPrint("config option:%s, input value:%s, directory not exist, create with return code:%d", option_name, input_value, code); + } +} + void tsReadIpConfig(SGlobalConfig *cfg, char *input_value) { uint32_t value = inet_addr(input_value); char * option = (char *)cfg->ptr; @@ -377,223 +426,353 @@ void tsInitConfigOption(SGlobalConfig *cfg, char *name, void *ptr, int8_t valTyp void tsInitGlobalConfig() { if (tsGlobalConfig != NULL) return; - tsGlobalConfig = (SGlobalConfig *)malloc(sizeof(SGlobalConfig) * TSDB_CFG_MAX_NUM); + tsGlobalConfig = (SGlobalConfig *) malloc(sizeof(SGlobalConfig) * TSDB_CFG_MAX_NUM); memset(tsGlobalConfig, 0, sizeof(SGlobalConfig) * TSDB_CFG_MAX_NUM); SGlobalConfig *cfg = tsGlobalConfig; // ip address - tsInitConfigOption(cfg++, "internalIp", tsInternalIp, TSDB_CFG_VTYPE_IPSTR, - TSDB_CFG_CTYPE_B_CONFIG, 0, 0, TSDB_IPv4ADDR_LEN, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "masterIp", tsMasterIp, TSDB_CFG_VTYPE_IPSTR, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_CLUSTER, + 0, 0, TSDB_IPv4ADDR_LEN, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "secondIp", tsSecondIp, TSDB_CFG_VTYPE_IPSTR, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_CLUSTER, + 0, 0, TSDB_IPv4ADDR_LEN, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "serverIp", tsServerIpStr, TSDB_CFG_VTYPE_IPSTR, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, 0, 0, TSDB_IPv4ADDR_LEN, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_LITE, + 0, 0, TSDB_IPv4ADDR_LEN, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "publicIp", tsPublicIp, TSDB_CFG_VTYPE_IPSTR, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLUSTER, + 0, 0, TSDB_IPv4ADDR_LEN, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "privateIp", tsPrivateIp, TSDB_CFG_VTYPE_IPSTR, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLUSTER, + 0, 0, TSDB_IPv4ADDR_LEN, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "internalIp", tsInternalIp, TSDB_CFG_VTYPE_IPSTR, + TSDB_CFG_CTYPE_B_CONFIG, + 0, 0, TSDB_IPv4ADDR_LEN, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "localIp", tsLocalIp, TSDB_CFG_VTYPE_IPSTR, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, 0, 0, TSDB_IPv4ADDR_LEN, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "httpIp", tsHttpIp, TSDB_CFG_VTYPE_IPSTR, TSDB_CFG_CTYPE_B_CONFIG, 0, 0, TSDB_IPv4ADDR_LEN, - TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 0, TSDB_IPv4ADDR_LEN, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "httpIp", tsHttpIp, TSDB_CFG_VTYPE_IPSTR, + TSDB_CFG_CTYPE_B_CONFIG, + 0, 0, TSDB_IPv4ADDR_LEN, TSDB_CFG_UTYPE_NONE); // port tsInitConfigOption(cfg++, "httpPort", &tsHttpPort, TSDB_CFG_VTYPE_SHORT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 1, 65535, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 1, 65535, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "mgmtShellPort", &tsMgmtShellPort, TSDB_CFG_VTYPE_SHORT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, 1, 65535, 0, - TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, + 1, 65535, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "vnodeShellPort", &tsVnodeShellPort, TSDB_CFG_VTYPE_SHORT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, 1, 65535, 0, - TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, + 1, 65535, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "mgmtVnodePort", &tsMgmtVnodePort, TSDB_CFG_VTYPE_SHORT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, + 1, 65535, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "vnodeVnodePort", &tsVnodeVnodePort, TSDB_CFG_VTYPE_SHORT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, + 1, 65535, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "mgmtMgmtPort", &tsMgmtMgmtPort, TSDB_CFG_VTYPE_SHORT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, + 1, 65535, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "mgmtSyncPort", &tsMgmtSyncPort, TSDB_CFG_VTYPE_SHORT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, + 1, 65535, 0, TSDB_CFG_UTYPE_NONE); // directory - tsInitConfigOption(cfg++, "configDir", configDir, TSDB_CFG_VTYPE_DIRECTORY, TSDB_CFG_CTYPE_B_CLIENT, 0, 0, - TSDB_FILENAME_LEN, TSDB_CFG_UTYPE_NONE); -#ifdef LINUX - tsInitConfigOption(cfg++, "dataDir", dataDir, TSDB_CFG_VTYPE_DIRECTORY, TSDB_CFG_CTYPE_B_CONFIG, 0, 0, - TSDB_FILENAME_LEN, TSDB_CFG_UTYPE_NONE); -#endif + tsInitConfigOption(cfg++, "configDir", configDir, TSDB_CFG_VTYPE_DIRECTORY, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 0, TSDB_FILENAME_LEN, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "logDir", logDir, TSDB_CFG_VTYPE_DIRECTORY, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, 0, 0, TSDB_FILENAME_LEN, - TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "scriptDir", scriptDir, TSDB_CFG_VTYPE_DIRECTORY, TSDB_CFG_CTYPE_B_CONFIG, 0, 0, - TSDB_FILENAME_LEN, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_LOG, + 0, 0, TSDB_FILENAME_LEN, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "scriptDir", scriptDir, TSDB_CFG_VTYPE_DIRECTORY, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 0, TSDB_FILENAME_LEN, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "dataDir", dataDir, TSDB_CFG_VTYPE_DIRECTORY, + TSDB_CFG_CTYPE_B_CONFIG, + 0, 0, TSDB_FILENAME_LEN, TSDB_CFG_UTYPE_NONE); // dnode configs tsInitConfigOption(cfg++, "numOfThreadsPerCore", &tsNumOfThreadsPerCore, TSDB_CFG_VTYPE_FLOAT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, 0, 10, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 10, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "ratioOfQueryThreads", &tsRatioOfQueryThreads, TSDB_CFG_VTYPE_FLOAT, - TSDB_CFG_CTYPE_B_CONFIG, 0.1, 0.9, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG, + 0.1, 0.9, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "numOfVnodesPerCore", &tsNumOfVnodesPerCore, TSDB_CFG_VTYPE_SHORT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 1, 64, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "numOfTotalVnodes", &tsNumOfTotalVnodes, TSDB_CFG_VTYPE_SHORT, TSDB_CFG_CTYPE_B_CONFIG, 0, - TSDB_MAX_VNODES, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "checkHeaderFile", &tsCheckHeaderFile, TSDB_CFG_VTYPE_SHORT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 0, 1, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 1, 64, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "numOfTotalVnodes", &tsNumOfTotalVnodes, TSDB_CFG_VTYPE_SHORT, + TSDB_CFG_CTYPE_B_CONFIG, + 0, TSDB_MAX_VNODES, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "tables", &tsSessionsPerVnode, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 4, 220000, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 4, 220000, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "cache", &tsCacheBlockSize, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 100, 1048576, 0, TSDB_CFG_UTYPE_BYTE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 100, 1048576, 0, TSDB_CFG_UTYPE_BYTE); tsInitConfigOption(cfg++, "rows", &tsRowsInFileBlock, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 200, 1048576, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 200, 1048576, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "fileBlockMinPercent", &tsFileBlockMinPercent, TSDB_CFG_VTYPE_FLOAT, - TSDB_CFG_CTYPE_B_CONFIG, 0, 1.0, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG, + 0, 1.0, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "ablocks", &tsAverageCacheBlocks, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 2, 128, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 2, 128, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "tblocks", &tsNumOfBlocksPerMeter, TSDB_CFG_VTYPE_SHORT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 32, 4096, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 32, 4096, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "numOfMPeers", &tsNumOfMPeers, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, + 1, 3, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "balanceInterval", &tsBalanceStartInterval, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, + 1, 30000, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "alternativeRole", &tsAlternativeRole, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLUSTER, + 0, 2, 0, TSDB_CFG_UTYPE_NONE); + // 0-any, 1-mgmt, 2-dnode // time - tsInitConfigOption(cfg++, "monitorInterval", &tsMonitorInterval, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG, 1, 600, - 0, TSDB_CFG_UTYPE_SECOND); + tsInitConfigOption(cfg++, "monitorInterval", &tsMonitorInterval, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG, + 1, 600, 0, TSDB_CFG_UTYPE_SECOND); + tsInitConfigOption(cfg++, "offlineThreshold", &tsOfflineThreshold, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, + 5, 7200000, 0, TSDB_CFG_UTYPE_SECOND); tsInitConfigOption(cfg++, "rpcTimer", &tsRpcTimer, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, 100, 3000, 0, TSDB_CFG_UTYPE_MS); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, + 100, 3000, 0, TSDB_CFG_UTYPE_MS); tsInitConfigOption(cfg++, "rpcMaxTime", &tsRpcMaxTime, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, 100, 7200, 0, TSDB_CFG_UTYPE_SECOND); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, + 100, 7200, 0, TSDB_CFG_UTYPE_SECOND); tsInitConfigOption(cfg++, "ctime", &tsCommitTime, TSDB_CFG_VTYPE_SHORT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 30, 40960, 0, TSDB_CFG_UTYPE_SECOND); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 30, 40960, 0, TSDB_CFG_UTYPE_SECOND); tsInitConfigOption(cfg++, "statusInterval", &tsStatusInterval, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 1, 10, 0, TSDB_CFG_UTYPE_SECOND); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 1, 10, 0, TSDB_CFG_UTYPE_SECOND); tsInitConfigOption(cfg++, "shellActivityTimer", &tsShellActivityTimer, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, 1, 120, 0, TSDB_CFG_UTYPE_SECOND); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, + 1, 120, 0, TSDB_CFG_UTYPE_SECOND); + tsInitConfigOption(cfg++, "vnodePeerHBTimer", &tsVnodePeerHBTimer, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, + 1, 10, 0, TSDB_CFG_UTYPE_SECOND); + tsInitConfigOption(cfg++, "mgmtPeerHBTimer", &tsMgmtPeerHBTimer, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, 1, 10, 0, + TSDB_CFG_UTYPE_SECOND); tsInitConfigOption(cfg++, "meterMetaKeepTimer", &tsMeterMetaKeepTimer, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, 1, 8640000, 0, TSDB_CFG_UTYPE_SECOND); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, + 1, 8640000, 0, TSDB_CFG_UTYPE_SECOND); tsInitConfigOption(cfg++, "metricMetaKeepTimer", &tsMetricMetaKeepTimer, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, 1, 8640000, 0, TSDB_CFG_UTYPE_SECOND); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, + 1, 8640000, 0, TSDB_CFG_UTYPE_SECOND); // mgmt configs + tsInitConfigOption(cfg++, "mgmtZone", tsMgmtZone, TSDB_CFG_VTYPE_STRING, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, + 0, 0, 16, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "maxAccounts", &tsMaxAccounts, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, + 1, 1000, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "maxUsers", &tsMaxUsers, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 1, 1000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxDbs", &tsMaxDbs, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 1, - 10000, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 1, 1000, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "maxDbs", &tsMaxDbs, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 1, 10000, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "maxTables", &tsMaxTables, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 1, 100000000, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 1, 100000000, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "maxDnodes", &tsMaxDnodes, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, + 1, 1000, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "maxVGroups", &tsMaxVGroups, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 1, 1000000, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 1, 1000000, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "minSlidingTime", &tsMinSlidingTime, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 10, 1000000, 0, TSDB_CFG_UTYPE_MS); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 10, 1000000, 0, TSDB_CFG_UTYPE_MS); tsInitConfigOption(cfg++, "minIntervalTime", &tsMinIntervalTime, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 10, 1000000, 0, TSDB_CFG_UTYPE_MS); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 10, 1000000, 0, TSDB_CFG_UTYPE_MS); tsInitConfigOption(cfg++, "maxStreamCompDelay", &tsMaxStreamComputDelay, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 10, 1000000000, 0, TSDB_CFG_UTYPE_MS); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 10, 1000000000, 0, TSDB_CFG_UTYPE_MS); tsInitConfigOption(cfg++, "maxFirstStreamCompDelay", &tsStreamCompStartDelay, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 1000, 1000000000, 0, TSDB_CFG_UTYPE_MS); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 1000, 1000000000, 0, TSDB_CFG_UTYPE_MS); tsInitConfigOption(cfg++, "retryStreamCompDelay", &tsStreamCompRetryDelay, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 10, 1000000000, 0, TSDB_CFG_UTYPE_MS); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 10, 1000000000, 0, TSDB_CFG_UTYPE_MS); tsInitConfigOption(cfg++, "clog", &tsCommitLog, TSDB_CFG_VTYPE_SHORT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 0, 1, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 0, 1, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "comp", &tsCompression, TSDB_CFG_VTYPE_SHORT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 0, 2, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 0, 2, 0, TSDB_CFG_UTYPE_NONE); // database configs tsInitConfigOption(cfg++, "days", &tsDaysPerFile, TSDB_CFG_VTYPE_SHORT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 1, 365, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 1, 365, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "keep", &tsDaysToKeep, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 1, 365000, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 1, 365000, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "replica", &tsReplications, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, + 1, 3, 0, TSDB_CFG_UTYPE_NONE); // login configs tsInitConfigOption(cfg++, "defaultDB", tsDefaultDB, TSDB_CFG_VTYPE_STRING, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, 0, 0, TSDB_DB_NAME_LEN, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 0, TSDB_DB_NAME_LEN, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "defaultUser", tsDefaultUser, TSDB_CFG_VTYPE_STRING, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, 0, 0, TSDB_USER_LEN, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 0, TSDB_USER_LEN, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "defaultPass", tsDefaultPass, TSDB_CFG_VTYPE_STRING, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_NOT_PRINT, 0, 0, - TSDB_PASSWORD_LEN, TSDB_CFG_UTYPE_NONE); - + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_NOT_PRINT, + 0, 0, TSDB_PASSWORD_LEN, TSDB_CFG_UTYPE_NONE); // socket type, udp by default tsInitConfigOption(cfg++, "sockettype", tsSocketType, TSDB_CFG_VTYPE_STRING, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW, 0, 0, 3, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW, + 0, 0, 3, TSDB_CFG_UTYPE_NONE); + + tsInitConfigOption(cfg++, "compressMsgSize", &tsCompressMsgSize, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW, + -1, 10000000, 0, TSDB_CFG_UTYPE_NONE); // locale & charset tsInitConfigOption(cfg++, "timezone", tsTimezone, TSDB_CFG_VTYPE_STRING, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, 0, 0, tListLen(tsTimezone), - TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 0, tListLen(tsTimezone), TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "locale", tsLocale, TSDB_CFG_VTYPE_STRING, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, 0, 0, tListLen(tsLocale), TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 0, tListLen(tsLocale), TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "charset", tsCharset, TSDB_CFG_VTYPE_STRING, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, 0, 0, tListLen(tsCharset), TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 0, tListLen(tsCharset), TSDB_CFG_UTYPE_NONE); // connect configs tsInitConfigOption(cfg++, "maxShellConns", &tsMaxShellConns, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 10, 50000000, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 10, 50000000, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "maxMeterConnections", &tsMaxMeterConnections, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 10, 50000000, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 10, 50000000, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "maxMgmtConnections", &tsMaxMgmtConnections, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 10, 50000000, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 10, 50000000, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "maxVnodeConnections", &tsMaxVnodeConnections, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 10, 50000000, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 10, 50000000, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "minimalLogDirGB", &tsMinimalLogDirGB, TSDB_CFG_VTYPE_FLOAT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 0.001, 10000000, 0, TSDB_CFG_UTYPE_GB); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 0.001, 10000000, 0, TSDB_CFG_UTYPE_GB); tsInitConfigOption(cfg++, "minimalTmpDirGB", &tsMinimalTmpDirGB, TSDB_CFG_VTYPE_FLOAT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 0.001, 10000000, 0, TSDB_CFG_UTYPE_GB); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 0.001, 10000000, 0, TSDB_CFG_UTYPE_GB); tsInitConfigOption(cfg++, "minimalDataDirGB", &tsMinimalDataDirGB, TSDB_CFG_VTYPE_FLOAT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 0.001, 10000000, 0, TSDB_CFG_UTYPE_GB); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 0.001, 10000000, 0, TSDB_CFG_UTYPE_GB); // module configs - tsInitConfigOption(cfg++, "enableHttp", &tsEnableHttpModule, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 0, 1, 1, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "enableMonitor", &tsEnableMonitorModule, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 0, 1, 1, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "mgmtEqualVnodeNum", &tsMgmtEqualVnodeNum, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, + 0, 1000, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "http", &tsEnableHttpModule, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 0, 1, 1, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "monitor", &tsEnableMonitorModule, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 0, 1, 1, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "monitorDbName", tsMonitorDbName, TSDB_CFG_VTYPE_STRING, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 0, 0, TSDB_DB_NAME_LEN, TSDB_CFG_UTYPE_NONE); // http configs - tsInitConfigOption(cfg++, "httpCacheSessions", &tsHttpCacheSessions, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG, 1, - 100000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "telegrafUseFieldNum", &tsTelegrafUseFieldNum, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 0, - 1, 1, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "httpMaxThreads", &tsHttpMaxThreads, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG, 1, - 1000000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "restfulRowLimit", &tsRestRowLimit, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG, 1, - 10000000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "httpEnableCompress", &tsHttpEnableCompress, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG, 0, - 1, 1, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "httpCacheSessions", &tsHttpCacheSessions, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG, + 1, 100000, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "httpEnableRecordSql", &tsHttpEnableRecordSql, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG, + 1, 100000, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "telegrafUseFieldNum", &tsTelegrafUseFieldNum, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, + 0, 1, 1, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "httpMaxThreads", &tsHttpMaxThreads, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG, + 1, 1000000, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "restfulRowLimit", &tsRestRowLimit, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG, + 1, 10000000, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "httpEnableCompress", &tsHttpEnableCompress, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG, + 0, 1, 1, TSDB_CFG_UTYPE_NONE); // debug flag tsInitConfigOption(cfg++, "numOfLogLines", &tsNumOfLogLines, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, 10000, 2000000000, 0, - TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, + 10000, 2000000000, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "asyncLog", &tsAsyncLog, TSDB_CFG_VTYPE_SHORT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, 0, 1, 0, - TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 1, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "debugFlag", &debugFlag, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, 0, 255, 0, - TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 255, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "mDebugFlag", &mdebugFlag, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG, 0, 255, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG, + 0, 255, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "dDebugFlag", &ddebugFlag, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG, 0, 255, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG, + 0, 255, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "sdbDebugFlag", &sdbDebugFlag, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG, 0, 255, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG, + 0, 255, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "rpcDebugFlag", &rpcDebugFlag, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, 0, 255, 0, - TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 255, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "tmrDebugFlag", &tmrDebugFlag, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, 0, 255, 0, - TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 255, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "cDebugFlag", &cdebugFlag, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, 0, 255, 0, - TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 255, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "jniDebugFlag", &jnidebugFlag, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, 0, 255, 0, - TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 255, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "odbcDebugFlag", &odbcdebugFlag, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, 0, 255, 0, - TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 255, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "uDebugFlag", &uDebugFlag, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, 0, 255, 0, - TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 255, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "httpDebugFlag", &httpDebugFlag, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG, 0, 255, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG, + 0, 255, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "monitorDebugFlag", &monitorDebugFlag, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG, 0, 255, 0, TSDB_CFG_UTYPE_NONE); + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG, + 0, 255, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "qDebugFlag", &qdebugFlag, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, 0, 255, 0, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 255, 0, TSDB_CFG_UTYPE_NONE); // version info - tsInitConfigOption(cfg++, "gitinfo", gitinfo, TSDB_CFG_VTYPE_STRING, TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, + tsInitConfigOption(cfg++, "gitinfo", gitinfo, TSDB_CFG_VTYPE_STRING, + TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, 0, 0, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "buildinfo", buildinfo, TSDB_CFG_VTYPE_STRING, - TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, 0, 0, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "version", version, TSDB_CFG_VTYPE_STRING, TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, + TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, + 0, 0, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "version", version, TSDB_CFG_VTYPE_STRING, + TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, 0, 0, 0, TSDB_CFG_UTYPE_NONE); tsGlobalConfigNum = (int)(cfg - tsGlobalConfig); @@ -665,8 +844,6 @@ bool tsReadGlobalConfig() { sprintf(fileName, "%s/taos.cfg", configDir); fp = fopen(fileName, "r"); if (fp == NULL) { - // printf("option file:%s not there, all options are set to system default\n", fileName); - // return; } else { line = NULL; while (!feof(fp)) { @@ -696,8 +873,30 @@ bool tsReadGlobalConfig() { fclose(fp); } + tsReadGlobalConfigSpec(); + + if (tsPrivateIp[0] == 0) { + taosGetPrivateIp(tsPrivateIp); + } + + if (tsPublicIp[0] == 0) { + strcpy(tsPublicIp, tsPrivateIp); + } + if (tsInternalIp[0] == 0) { - taosGetPrivateIp(tsInternalIp); + strcpy(tsInternalIp, tsPrivateIp); + } + + if (tsLocalIp[0] == 0) { + strcpy(tsLocalIp, tsPrivateIp); + } + + if (tsMasterIp[0] == 0) { + strcpy(tsMasterIp, tsPrivateIp); + } + + if (tsSecondIp[0] == 0) { + strcpy(tsSecondIp, tsMasterIp); } taosGetSystemInfo(); @@ -713,6 +912,15 @@ bool tsReadGlobalConfig() { tsNumOfCores = 1; } + if (strlen(tsPrivateIp) == 0) { + pError("privateIp is null"); + return false; + } + + if (tscEmbedded) { + strcpy(tsLocalIp, tsPrivateIp); + } + tsVersion = 0; for (int i = 0; i < 10; i++) { if (version[i] >= '0' && version[i] <= '9') { @@ -750,7 +958,19 @@ int tsCfgDynamicOptions(char *msg) { if (strncasecmp(option, cfg->option, olen) != 0) continue; *((int *)cfg->ptr) = vint; - if (strncasecmp(cfg->option, "debugFlag", olen) == 0) tsSetAllDebugFlag(); + if (strncasecmp(cfg->option, "monitor", olen) == 0) { + if (0 == vint) { + if(stopMonitor) (void)(*stopMonitor)(); + } else { + if(startMonitor) (*startMonitor)(); + } + return code; + } + + if (strncasecmp(cfg->option, "debugFlag", olen) == 0) { + tsSetAllDebugFlag(); + } + return code; } @@ -783,7 +1003,17 @@ void tsPrintGlobalConfig() { SGlobalConfig *cfg = tsGlobalConfig + i; if (tscEmbedded == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - + if (cfg->cfgType & TSDB_CFG_CTYPE_B_LITE) { +#ifdef CLUSTER + continue; +#endif + } + if (cfg->cfgType & TSDB_CFG_CTYPE_B_CLUSTER) { +#ifndef CLUSTER + continue; +#endif + } + int optionLen = (int)strlen(cfg->option); int blankLen = TSDB_CFG_PRINT_LEN - optionLen; blankLen = blankLen < 0 ? 0 : blankLen; @@ -814,9 +1044,8 @@ void tsPrintGlobalConfig() { break; } } -#ifdef LINUX - pPrint(" dataDir: %s", dataDir); -#endif + + tsPrintGlobalConfigSpec(); tsPrintOsInfo(); @@ -910,3 +1139,13 @@ void tsSetTimeZone() { pPrint("timezone format changed to %s", tsTimezone); } + +#ifndef CLUSTER + +bool tsReadGlobalConfigSpec() { return true; } + +void tsPrintGlobalConfigSpec() { + pPrint(" dataDir: %s", dataDir); +} + +#endif \ No newline at end of file diff --git a/src/util/src/tidpool.c b/src/util/src/tidpool.c index cf999974a8..a9cc78a026 100644 --- a/src/util/src/tidpool.c +++ b/src/util/src/tidpool.c @@ -150,3 +150,38 @@ void taosIdPoolSetFreeList(void *handle) { } } } + +int taosUpdateIdPool(id_pool_t *handle, int maxId) { + id_pool_t *pIdPool = (id_pool_t*)handle; + if (maxId <= pIdPool->maxId) { + return -1; + } + + int *idList, i; + idList = (int *)malloc(sizeof(int) * (size_t)maxId); + if (idList == NULL) { + return -1; + } + for (i = 1; i < maxId; ++i) { + idList[i - 1] = i; + } + + if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex"); + + memcpy(idList, pIdPool->freeList, sizeof(int) * (size_t)pIdPool->maxId); + pIdPool->numOfFree += (maxId - pIdPool->maxId); + pIdPool->maxId = maxId; + + int *oldIdList = pIdPool->freeList; + pIdPool->freeList = idList; + free(oldIdList); + + if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex"); + + return 0; +} + +int taosIdPoolMaxSize(void *handle) { + id_pool_t *pIdPool = (id_pool_t*)handle; + return pIdPool->maxId; +} \ No newline at end of file diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index 821bcd0e64..02e30be33a 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -41,7 +41,7 @@ #define MAX_LOGLINE_DUMP_CONTENT_SIZE (MAX_LOGLINE_DUMP_SIZE - 100) #define LOG_FILE_NAME_LEN 300 -#define TSDB_DEFAULT_LOG_BUF_SIZE (64 * 1024) // 10K +#define TSDB_DEFAULT_LOG_BUF_SIZE (512 * 1024) // 512K #define TSDB_MIN_LOG_BUF_SIZE 1024 // 1K #define TSDB_MAX_LOG_BUF_SIZE (1024 * 1024) // 1M #define TSDB_DEFAULT_LOG_BUF_UNIT 1024 // 1K @@ -61,7 +61,7 @@ typedef struct { uint32_t uDebugFlag = 131; // all the messages short tsAsyncLog = 1; -static SLogBuff *logHandle; +static SLogBuff *logHandle = NULL; static int taosLogFileNum = 1; static int taosLogMaxLines = 0; static int taosLogLines = 0; diff --git a/src/util/src/tmd5.c b/src/util/src/tmd5.c index afbd4d9dcb..90b41dacf4 100644 --- a/src/util/src/tmd5.c +++ b/src/util/src/tmd5.c @@ -90,10 +90,7 @@ static uint8_t PADDING[64] = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x void MD5Init(MD5_CTX *mdContext) { memset(mdContext, 0, sizeof(MD5_CTX)); - mdContext->i[0] = mdContext->i[1] = (uint32_t)0; - - /* Load magic initialization constants. - */ + /* Load magic initialization constants. */ mdContext->buf[0] = (uint32_t)0x67452301; mdContext->buf[1] = (uint32_t)0xefcdab89; mdContext->buf[2] = (uint32_t)0x98badcfe; diff --git a/src/util/src/tmem.c b/src/util/src/tmem.c new file mode 100644 index 0000000000..c2bb836635 --- /dev/null +++ b/src/util/src/tmem.c @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include + +#include "tlog.h" + +extern int32_t taosGetTimestampSec(); +static int32_t startTime = 0; +static int64_t m_curLimit = 100*1024; + +bool isMallocMem(unsigned int size, char* _func) { + if (0 == startTime) { + startTime = taosGetTimestampSec(); + return true; + } else { + int32_t currentTime = taosGetTimestampSec(); + if (currentTime - startTime < 10) return true; + } + + if (size > m_curLimit) { + if (3 == rand() % 20) { + pTrace("====no alloc mem in func: %s, size:%d", _func, size); + return false; + } + } + + return true; +} + +void* taos_malloc(unsigned int size, char* _func) { + + if (false == isMallocMem(size, _func)) { + return NULL; + } + + void *p = NULL; + p = malloc(size); + return p; +} + +void* taos_calloc(unsigned int num, unsigned int size, char* _func) { + + if (false == isMallocMem(size, _func)) { + return NULL; + } + + void *p = NULL; + p = calloc(num, size); + return p; +} + +void* taos_realloc(void* ptr, unsigned int size, char* _func) { + + if (false == isMallocMem(size, _func)) { + return NULL; + } + + void *p = NULL; + p = realloc(ptr, size); + return p; +} + +void taos_free(void* ptr) { + free(ptr); +} + diff --git a/src/util/src/tmempool.c b/src/util/src/tmempool.c index 6276a3ed6c..f2cf31783c 100644 --- a/src/util/src/tmempool.c +++ b/src/util/src/tmempool.c @@ -56,9 +56,9 @@ mpool_h taosMemPoolInit(int numOfBlock, int blockSize) { if (pool_p->pool == NULL || pool_p->freeList == NULL) { pError("failed to allocate memory\n"); - free(pool_p->freeList); - free(pool_p->pool); - free(pool_p); + tfree(pool_p->freeList); + tfree(pool_p->pool); + tfree(pool_p); return NULL; } diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c new file mode 100644 index 0000000000..68228d377a --- /dev/null +++ b/src/util/src/tnote.c @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "os.h" +#include "tutil.h" +#include "tglobalcfg.h" + +#define MAX_NOTE_LINE_SIZE 66000 +#define NOTE_FILE_NAME_LEN 300 + +static int taosNoteFileNum = 1; +static int taosNoteMaxLines = 0; +static int taosNoteLines = 0; +static char taosNoteName[NOTE_FILE_NAME_LEN]; +static int taosNoteFlag = 0; +static int taosNoteFd = -1; +static int taosNoteOpenInProgress = 0; +static pthread_mutex_t taosNoteMutex; +void taosNotePrint(const char * const format, ...); +int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum); + +void taosInitNote(int numOfNoteLines, int maxNotes) +{ + char temp[128] = { 0 }; + sprintf(temp, "%s/taosnote", logDir); + if (taosOpenNoteWithMaxLines(temp, numOfNoteLines, maxNotes) < 0) + fprintf(stderr, "failed to init note file\n"); + + taosNotePrint("=================================================="); + taosNotePrint("=================== new note ==================="); + taosNotePrint("=================================================="); +} + +void taosCloseNoteByFd(int oldFd); +bool taosLockNote(int fd) +{ + if (fd < 0) return false; + + if (taosNoteFileNum > 1) { + int ret = (int)(flock(fd, LOCK_EX | LOCK_NB)); + if (ret == 0) { + return true; + } + } + + return false; +} + +void taosUnLockNote(int fd) +{ + if (fd < 0) return; + + if (taosNoteFileNum > 1) { + flock(fd, LOCK_UN | LOCK_NB); + } +} + +void *taosThreadToOpenNewNote(void *param) +{ + char name[NOTE_FILE_NAME_LEN]; + + taosNoteFlag ^= 1; + taosNoteLines = 0; + sprintf(name, "%s.%d", taosNoteName, taosNoteFlag); + + umask(0); + + int fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + taosLockNote(fd); + lseek(fd, 0, SEEK_SET); + + int oldFd = taosNoteFd; + taosNoteFd = fd; + taosNoteLines = 0; + taosNoteOpenInProgress = 0; + taosNotePrint("=============== new note is opened ============="); + + taosCloseNoteByFd(oldFd); + return NULL; +} + +int taosOpenNewNote() +{ + pthread_mutex_lock(&taosNoteMutex); + + if (taosNoteLines > taosNoteMaxLines && taosNoteOpenInProgress == 0) { + taosNoteOpenInProgress = 1; + + taosNotePrint("=============== open new note =================="); + pthread_t pattern; + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + + pthread_create(&pattern, &attr, taosThreadToOpenNewNote, NULL); + pthread_attr_destroy(&attr); + } + + pthread_mutex_unlock(&taosNoteMutex); + + return taosNoteFd; +} + +bool taosCheckNoteIsOpen(char *noteName) +{ + int exist = access(noteName, F_OK); + if (exist != 0) { + return false; + } + + int fd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + if (fd < 0) { + fprintf(stderr, "failed to open note:%s reason:%s\n", noteName, strerror(errno)); + return true; + } + + if (taosLockNote(fd)) { + taosUnLockNote(fd); + close(fd); + return false; + } + else { + close(fd); + return true; + } +} + +void taosGetNoteName(char *fn) +{ + if (taosNoteFileNum > 1) { + for (int i = 0; i < taosNoteFileNum; i++) { + char fileName[NOTE_FILE_NAME_LEN]; + + sprintf(fileName, "%s%d.0", fn, i); + bool file1open = taosCheckNoteIsOpen(fileName); + + sprintf(fileName, "%s%d.1", fn, i); + bool file2open = taosCheckNoteIsOpen(fileName); + + if (!file1open && !file2open) { + sprintf(taosNoteName, "%s%d", fn, i); + return; + } + } + } + + strcpy(taosNoteName, fn); +} + +int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum) +{ + char name[NOTE_FILE_NAME_LEN] = "\0"; + struct stat notestat0, notestat1; + int size; + + taosNoteMaxLines = maxLines; + taosNoteFileNum = maxNoteNum; + taosGetNoteName(fn); + + strcpy(name, fn); + strcat(name, ".0"); + + // if none of the note files exist, open 0, if both exists, open the old one + if (stat(name, ¬estat0) < 0) { + taosNoteFlag = 0; + } + else { + strcpy(name, fn); + strcat(name, ".1"); + if (stat(name, ¬estat1) < 0) { + taosNoteFlag = 1; + } + else { + taosNoteFlag = (notestat0.st_mtime > notestat1.st_mtime) ? 0 : 1; + } + } + + sprintf(name, "%s.%d", taosNoteName, taosNoteFlag); + pthread_mutex_init(&taosNoteMutex, NULL); + + umask(0); + taosNoteFd = open(name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + + if (taosNoteFd < 0) { + fprintf(stderr, "failed to open note file:%s reason:%s\n", name, strerror(errno)); + return -1; + } + taosLockNote(taosNoteFd); + + // only an estimate for number of lines + struct stat filestat; + fstat(taosNoteFd, &filestat); + size = (int)filestat.st_size; + taosNoteLines = size / 60; + + lseek(taosNoteFd, 0, SEEK_END); + + return 0; +} + +void taosNotePrint(const char * const format, ...) +{ + va_list argpointer; + char buffer[MAX_NOTE_LINE_SIZE]; + int len; + struct tm Tm, *ptm; + struct timeval timeSecs; + time_t curTime; + + gettimeofday(&timeSecs, NULL); + curTime = timeSecs.tv_sec; + ptm = localtime_r(&curTime, &Tm); + len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int)timeSecs.tv_usec); + + va_start(argpointer, format); + len += vsnprintf(buffer + len, MAX_NOTE_LINE_SIZE - len, format, argpointer); + va_end(argpointer); + + if (len >= MAX_NOTE_LINE_SIZE) len = MAX_NOTE_LINE_SIZE - 2; + + buffer[len++] = '\n'; + buffer[len] = 0; + + if (taosNoteFd >= 0) { + twrite(taosNoteFd, buffer, (unsigned int)len); + + if (taosNoteMaxLines > 0) { + taosNoteLines++; + if ((taosNoteLines > taosNoteMaxLines) && (taosNoteOpenInProgress == 0)) + taosOpenNewNote(); + } + } +} + +void taosCloseNote() +{ + taosCloseNoteByFd(taosNoteFd); +} + +void taosCloseNoteByFd(int fd) +{ + if (fd >= 0) { + taosUnLockNote(fd); + close(fd); + } +} diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index d7779a94af..72469fa75b 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -23,13 +23,19 @@ #include "tskiplist.h" #include "tutil.h" -void recordNodeEachLevel(tSkipList *pSkipList, int32_t nLevel); - -int32_t getSkipListNodeLevel(tSkipList *pSkipList); +static FORCE_INLINE void recordNodeEachLevel(tSkipList *pSkipList, int32_t nLevel) { // record link count in each level + for (int32_t i = 0; i < nLevel; ++i) { + pSkipList->state.nLevelNodeCnt[i]++; + } +} -void tSkipListDoInsert(tSkipList *pSkipList, tSkipListNode **forward, int32_t nLevel, tSkipListNode *pNode); +static FORCE_INLINE void removeNodeEachLevel(tSkipList *pSkipList, int32_t nLevel) { + for (int32_t i = 0; i < nLevel; ++i) { + pSkipList->state.nLevelNodeCnt[i]--; + } +} -static int32_t getSkipListNodeRandomHeight(tSkipList *pSkipList) { +static FORCE_INLINE int32_t getSkipListNodeRandomHeight(tSkipList *pSkipList) { const uint32_t factor = 4; int32_t n = 1; @@ -40,6 +46,21 @@ static int32_t getSkipListNodeRandomHeight(tSkipList *pSkipList) { return n; } +static FORCE_INLINE int32_t getSkipListNodeLevel(tSkipList *pSkipList) { + int32_t nLevel = getSkipListNodeRandomHeight(pSkipList); + if (pSkipList->nSize == 0) { + nLevel = 1; + pSkipList->nLevel = 1; + } else { + if (nLevel > pSkipList->nLevel && pSkipList->nLevel < pSkipList->nMaxLevel) { + nLevel = (++pSkipList->nLevel); + } + } + return nLevel; +} + +void tSkipListDoInsert(tSkipList *pSkipList, tSkipListNode **forward, int32_t nLevel, tSkipListNode *pNode); + void tSkipListDoRecordPut(tSkipList *pSkipList) { const int32_t MAX_RECORD_NUM = 1000; @@ -96,16 +117,11 @@ int32_t compareStrVal(const void *pLeft, const void *pRight) { return 0; } - /* - * handle only one-side bound compare situation, there is only lower bound or - * only - * upper bound - */ + //handle only one-side bound compare situation, there is only lower bound or only upper bound if (pL->nLen == -1) { return 1; // no lower bound, lower bound is minimum, always return -1; } else if (pR->nLen == -1) { - return -1; // no upper bound, upper bound is maximum situation, always - // return 1; + return -1; // no upper bound, upper bound is maximum situation, always return 1; } int32_t ret = strcmp(((tSkipListKey *)pLeft)->pz, ((tSkipListKey *)pRight)->pz); @@ -125,15 +141,11 @@ int32_t compareWStrVal(const void *pLeft, const void *pRight) { return 0; } - /* - * handle only one-side bound compare situation, - * there is only lower bound or only upper bound - */ + //handle only one-side bound compare situation, there is only lower bound or only upper bound if (pL->nLen == -1) { return 1; // no lower bound, lower bound is minimum, always return -1; } else if (pR->nLen == -1) { - return -1; // no upper bound, upper bound is maximum situation, always - // return 1; + return -1; // no upper bound, upper bound is maximum situation, always return 1; } int32_t ret = wcscmp(((tSkipListKey *)pLeft)->wpz, ((tSkipListKey *)pRight)->wpz); @@ -302,7 +314,7 @@ static tSkipListNode *tSkipListCreateNode(void *pData, const tSkipListKey *pKey, tSkipListKey tSkipListCreateKey(int32_t type, char *val, size_t keyLength) { tSkipListKey k = {0}; - tVariantCreateB(&k, val, (uint32_t) keyLength, (uint32_t) type); + tVariantCreateFromBinary(&k, val, (uint32_t) keyLength, (uint32_t) type); return k; } @@ -394,31 +406,6 @@ void tSkipListDoInsert(tSkipList *pSkipList, tSkipListNode **forward, int32_t nL } } -int32_t getSkipListNodeLevel(tSkipList *pSkipList) { - int32_t nLevel = getSkipListNodeRandomHeight(pSkipList); - if (pSkipList->nSize == 0) { - nLevel = 1; - pSkipList->nLevel = 1; - } else { - if (nLevel > pSkipList->nLevel && pSkipList->nLevel < pSkipList->nMaxLevel) { - nLevel = (++pSkipList->nLevel); - } - } - return nLevel; -} - -void recordNodeEachLevel(tSkipList *pSkipList, int32_t nLevel) { // record link count in each level - for (int32_t i = 0; i < nLevel; ++i) { - pSkipList->state.nLevelNodeCnt[i]++; - } -} - -void removeNodeEachLevel(tSkipList *pSkipList, int32_t nLevel) { - for (int32_t i = 0; i < nLevel; ++i) { - pSkipList->state.nLevelNodeCnt[i]--; - } -} - tSkipListNode *tSkipListGetOne(tSkipList *pSkipList, tSkipListKey *pKey) { int32_t sLevel = pSkipList->nLevel - 1; int32_t ret = -1; @@ -548,11 +535,16 @@ static tSkipListNode *tSkipListParQuery(tSkipList *pSkipList, tSkipListKey *pKey int32_t tSkipListIterateList(tSkipList *pSkipList, tSkipListNode ***pRes, bool (*fp)(tSkipListNode *, void *), void *param) { + (*pRes) = (tSkipListNode **)calloc(1, POINTER_BYTES * pSkipList->nSize); + if (NULL == *pRes) { + pError("error skiplist %p, malloc failed", pSkipList); + return -1; + } + pthread_rwlock_rdlock(&pSkipList->lock); - - (*pRes) = (tSkipListNode **)malloc(POINTER_BYTES * pSkipList->nSize); tSkipListNode *pStartNode = pSkipList->pHead.pForward[0]; int32_t num = 0; + for (int32_t i = 0; i < pSkipList->nSize; ++i) { if (pStartNode == NULL) { pError("error skiplist %p, required length:%d, actual length:%d", pSkipList, pSkipList->nSize, i - 1); @@ -568,10 +560,60 @@ int32_t tSkipListIterateList(tSkipList *pSkipList, tSkipListNode ***pRes, bool ( pStartNode = pStartNode->pForward[0]; } + pthread_rwlock_unlock(&pSkipList->lock); + + if (num == 0) { + free(*pRes); + *pRes = NULL; + } else if (num < pSkipList->nSize) { // free unused memory + char* tmp = realloc((*pRes), num * POINTER_BYTES); + assert(tmp != NULL); + + *pRes = tmp; + } + return num; } +int32_t tSkipListIteratorReset(tSkipList *pSkipList, SSkipListIterator* iter) { + if (pSkipList == NULL) { + return -1; + } + + iter->pSkipList = pSkipList; + + pthread_rwlock_rdlock(&pSkipList->lock); + iter->cur = NULL;//pSkipList->pHead.pForward[0]; + iter->num = pSkipList->nSize; + pthread_rwlock_unlock(&pSkipList->lock); + + return 0; +} + +bool tSkipListIteratorNext(SSkipListIterator* iter) { + if (iter->num == 0 || iter->pSkipList == NULL) { + return false; + } + + tSkipList* pSkipList = iter->pSkipList; + + pthread_rwlock_rdlock(&pSkipList->lock); + if (iter->cur == NULL) { + iter->cur = pSkipList->pHead.pForward[0]; + } else { + iter->cur = iter->cur->pForward[0]; + } + + pthread_rwlock_unlock(&pSkipList->lock); + + return iter->cur != NULL; +} + +tSkipListNode* tSkipListIteratorGet(SSkipListIterator* iter) { + return iter->cur; +} + int32_t tSkipListRangeQuery(tSkipList *pSkipList, tSKipListQueryCond *pCond, tSkipListNode ***pRes) { pSkipList->state.queryCount++; tSkipListNode *pStart = tSkipListParQuery(pSkipList, &pCond->lowerBnd, pCond->lowerBndRelOptr); diff --git a/src/util/src/tstatus.c b/src/util/src/tstatus.c index 8539944dfb..570ba547eb 100644 --- a/src/util/src/tstatus.c +++ b/src/util/src/tstatus.c @@ -16,3 +16,7 @@ char* sdbDnodeStatusStr[] = {"offline", "creating", "unsynced", "slave", "master", "ready"}; char* sdbDnodeBalanceStateStr[] = {"balanced", "balancing", "offline removing", "shell removing"}; + +char* sdbVnodeSyncStatusStr[] = {"init", "syncing", "sync_cache", "sync_file"}; + +char* sdbVnodeDropStateStr[] = {"ready", "dropping"}; diff --git a/src/util/src/tstoken.c b/src/util/src/tstoken.c index a29a06a9be..e88d3bada6 100644 --- a/src/util/src/tstoken.c +++ b/src/util/src/tstoken.c @@ -22,69 +22,79 @@ #include "os.h" #include "shash.h" #include "tstoken.h" +void shiftStr(char *dst, char *src); -static const char operator[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +static char operator[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '$', '%', '&', 0, '(', ')', '*', '+', 0, '-', 0, '/', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '<', '=', '>', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '[', 0, ']', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '|', 0, 0, 0}; -static const char delimiter[] = { +static char delimiter[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ',', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ';', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; -char *tscGetTokenDelimiter(char *string, char **token, int *tokenLen, const char *delimiters) { - while ((*string != 0) && strchr(delimiters, *string)) { - ++string; +bool isCharInDelimiter(char c, char *delimiter) { + for (int i = 0; i < strlen(delimiter); i++) { + if (delimiter[i] == c) return true; + } + return false; +} + +char *tscGetTokenDelimiter(char *string, char **token, int *tokenLen, char *delimiters) { + while (*string != 0) { + if (isCharInDelimiter(*string, delimiters)) { + ++string; + } else { + break; + } } *token = string; char *str = string; - while ((*str != 0) && (strchr(delimiters, *str) == NULL)) { - ++str; + *tokenLen = 0; + while (*str != 0) { + if (!isCharInDelimiter(*str, delimiters)) { + *tokenLen = *tokenLen + 1; + str++; + } else { + break; + } } - *tokenLen = str - string; - return string; } -static bool isOperator(char c) { - return (c < 0) ? false : (operator[c] != 0); -} - -static bool isDelimiter(char c) { - return (c < 0) ? false : (delimiter[c] != 0); -} - char *tscGetToken(char *string, char **token, int *tokenLen) { char quote = 0; while (*string != 0) { - if (isDelimiter(*string)) { + if (delimiter[*string]) { ++string; } else { break; } } + char quotaChar = 0; if (*string == '\'' || *string == '\"') { - quote = *string; + quote = 1; + quotaChar = *string; string++; } *token = string; /* not in string, return token */ - if (quote == 0 && isOperator(*string)) { + if (*string > 0 && operator[*string] && quote == 0) { string++; /* handle the case: insert into tabx using stable1 tags(-1)/tags(+1) * values(....) */ - if (isOperator(*string) &&(*string != '(' && *string != ')' && *string != '-' && *string != '+')) + if (operator[*string] &&(*string != '(' && *string != ')' && *string != '-' && *string != '+')) *tokenLen = 2; else *tokenLen = 1; @@ -93,24 +103,28 @@ char *tscGetToken(char *string, char **token, int *tokenLen) { while (*string != 0) { if (quote) { - if (*string == '\'' || *string == '"') { - // handle escape situation, " and ' should not be eliminated - if (*(string - 1) == '\\') { - shiftStr(string - 1, string); - continue; - } else if (*string == quote) { + // handle escape situation: '\"', the " should not be eliminated + if (*string == quotaChar) { + if (*(string - 1) != '\\') { break; + } else { + shiftStr(string - 1, string); } + } else { + ++string; } - } else if (isDelimiter(*string) || isOperator(*string)) { - break; + } else { + if (delimiter[*string]) break; + + if (operator[*string]) break; + + ++string; } - ++string; } *tokenLen = (int)(string - *token); - if (quote && *string != 0) { + if (quotaChar != 0 && *string != 0 && *(string + 1) != 0) { return string + 1; } else { return string; @@ -122,7 +136,7 @@ void shiftStr(char *dst, char *src) { do { dst[i] = src[i]; i++; - } while (!isDelimiter(src[i])); + } while (delimiter[src[i]] == 0); src[i - 1] = ' '; } diff --git a/src/util/src/tstrbuild.c b/src/util/src/tstrbuild.c new file mode 100644 index 0000000000..6fb970bd6e --- /dev/null +++ b/src/util/src/tstrbuild.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tstrbuild.h" +#include +#include +#include + +void taosStringBuilderEnsureCapacity(SStringBuilder* sb, size_t size) { + size += sb->pos; + if (size > sb->size) { + size *= 2; + void* tmp = realloc(sb->buf, size); + if (tmp == NULL) { + longjmp(sb->jb, 1); + } + sb->buf = (char*)tmp; + sb->size = size; + } +} + +char* taosStringBuilderGetResult(SStringBuilder* sb, size_t* len) { + taosStringBuilderEnsureCapacity(sb, 1); + sb->buf[sb->pos] = 0; + if (len != NULL) { + *len = sb->pos; + } + return sb->buf; +} + +void taosStringBuilderDestroy(SStringBuilder* sb) { + free(sb->buf); + sb->buf = NULL; + sb->pos = 0; + sb->size = 0; +} + +void taosStringBuilderAppend(SStringBuilder* sb, const void* data, size_t len) { + taosStringBuilderEnsureCapacity(sb, len); + memcpy(sb->buf + sb->pos, data, len); + sb->pos += len; +} + +void taosStringBuilderAppendChar(SStringBuilder* sb, char c) { + taosStringBuilderEnsureCapacity(sb, 1); + sb->buf[sb->pos++] = c; +} + +void taosStringBuilderAppendStringLen(SStringBuilder* sb, const char* str, size_t len) { + taosStringBuilderEnsureCapacity(sb, len); + memcpy(sb->buf + sb->pos, str, len); + sb->pos += len; +} + +void taosStringBuilderAppendString(SStringBuilder* sb, const char* str) { + taosStringBuilderAppendStringLen(sb, str, strlen(str)); +} + +void taosStringBuilderAppendNull(SStringBuilder* sb) { taosStringBuilderAppendStringLen(sb, "null", 4); } + +void taosStringBuilderAppendInteger(SStringBuilder* sb, int64_t v) { + char buf[64]; + size_t len = sprintf(buf, "%lld", v); + taosStringBuilderAppendStringLen(sb, buf, len); +} + +void taosStringBuilderAppendDouble(SStringBuilder* sb, double v) { + char buf[64]; + size_t len = sprintf(buf, "%.9lf", v); + taosStringBuilderAppendStringLen(sb, buf, len); +} diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index 5c7fb82d93..5c726390e7 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -153,6 +153,13 @@ void *taosTmrInit(int maxNumOfTmrs, int resolution, int longest, char *label) { if (pCtrl->numOfPeriods < 10) pCtrl->numOfPeriods = 10; pCtrl->tmrList = (tmr_list_t *)malloc(sizeof(tmr_list_t) * pCtrl->numOfPeriods); + if (pCtrl->tmrList == NULL) { + tmrError("%s failed to allocate(size:%d) mem for tmrList", label, sizeof(tmr_list_t) * pCtrl->numOfPeriods); + tmrMemPoolCleanUp(pCtrl->poolHandle); + taosTmrCleanUp(pCtrl); + return NULL; + } + for (int i = 0; i < pCtrl->numOfPeriods; i++) { pCtrl->tmrList[i].head = NULL; pCtrl->tmrList[i].count = 0; @@ -292,6 +299,44 @@ tmr_h taosTmrStart(void (*fp)(void *, void *), int mseconds, void *param1, void return (tmr_h)pObj; } +void taosTmrStop(tmr_h timerId) { + tmr_obj_t * pObj; + tmr_list_t *pList; + tmr_ctrl_t *pCtrl; + + pObj = (tmr_obj_t *)timerId; + if (pObj == NULL) return; + + pCtrl = pObj->pCtrl; + if (pCtrl == NULL) return; + + if (pthread_mutex_lock(&pCtrl->mutex) != 0) + tmrError("%s mutex lock failed, reason:%s", pCtrl->label, strerror(errno)); + + if (pObj->timerId == timerId) { + pList = &(pCtrl->tmrList[pObj->index]); + if (pObj->prev) { + pObj->prev->next = pObj->next; + } else { + pList->head = pObj->next; + } + + if (pObj->next) { + pObj->next->prev = pObj->prev; + } + + pList->count--; + pObj->timerId = NULL; + pCtrl->numOfTmrs--; + + tmrTrace("%s %p, timer stopped, fp:%p, tmr_h:%p, total:%d", pCtrl->label, pObj->param1, pObj->fp, pObj, + pCtrl->numOfTmrs); + tmrMemPoolFree(pCtrl->poolHandle, (char *)(pObj)); + } + + pthread_mutex_unlock(&pCtrl->mutex); +} + void taosTmrStopA(tmr_h *timerId) { tmr_obj_t * pObj; tmr_list_t *pList; @@ -462,16 +507,16 @@ mpool_h tmrMemPoolInit(int numOfBlock, int blockSize) { pool_p->blockSize = blockSize; pool_p->numOfBlock = numOfBlock; pool_p->pool = (char *)malloc(blockSize * numOfBlock); - memset(pool_p->pool, 0, blockSize * numOfBlock); pool_p->freeList = (int *)malloc(sizeof(int) * numOfBlock); if (pool_p->pool == NULL || pool_p->freeList == NULL) { tmrError("failed to allocate memory\n"); - free(pool_p->freeList); - free(pool_p->pool); + tfree(pool_p->freeList); + tfree(pool_p->pool); free(pool_p); return NULL; } + memset(pool_p->pool, 0, blockSize * numOfBlock); for (i = 0; i < pool_p->numOfBlock; ++i) pool_p->freeList[i] = i; diff --git a/src/util/src/ttokenizer.c b/src/util/src/ttokenizer.c index fad7299ae6..99f0f5bb91 100644 --- a/src/util/src/ttokenizer.c +++ b/src/util/src/ttokenizer.c @@ -22,200 +22,211 @@ #include "tsql.h" #include "tutil.h" -//All the keywords of the SQL language are stored in a hash table +// All the keywords of the SQL language are stored in a hash table typedef struct SKeyword { - const char* name; // The keyword name - uint8_t type; // type - uint8_t len; // length + const char* name; // The keyword name + uint8_t type; // type + uint8_t len; // length } SKeyword; +// keywords in sql string static SKeyword keywordTable[] = { - {"ID", TK_ID}, - {"BOOL", TK_BOOL}, - {"TINYINT", TK_TINYINT}, - {"SMALLINT", TK_SMALLINT}, - {"INTEGER", TK_INTEGER}, - {"INT", TK_INTEGER}, - {"BIGINT", TK_BIGINT}, - {"FLOAT", TK_FLOAT}, - {"DOUBLE", TK_DOUBLE}, - {"STRING", TK_STRING}, - {"TIMESTAMP", TK_TIMESTAMP}, - {"BINARY", TK_BINARY}, - {"NCHAR", TK_NCHAR}, - {"OR", TK_OR}, - {"AND", TK_AND}, - {"NOT", TK_NOT}, - {"EQ", TK_EQ}, - {"NE", TK_NE}, - {"ISNULL", TK_ISNULL}, - {"NOTNULL", TK_NOTNULL}, - {"IS", TK_IS}, - {"LIKE", TK_LIKE}, - {"GLOB", TK_GLOB}, - {"BETWEEN", TK_BETWEEN}, - {"IN", TK_IN}, - {"GT", TK_GT}, - {"GE", TK_GE}, - {"LT", TK_LT}, - {"LE", TK_LE}, - {"BITAND", TK_BITAND}, - {"BITOR", TK_BITOR}, - {"LSHIFT", TK_LSHIFT}, - {"RSHIFT", TK_RSHIFT}, - {"PLUS", TK_PLUS}, - {"MINUS", TK_MINUS}, - {"DIVIDE", TK_DIVIDE}, - {"TIMES", TK_TIMES}, - {"STAR", TK_STAR}, - {"SLASH", TK_SLASH}, - {"REM ", TK_REM}, - {"CONCAT", TK_CONCAT}, - {"UMINUS", TK_UMINUS}, - {"UPLUS", TK_UPLUS}, - {"BITNOT", TK_BITNOT}, - {"SHOW", TK_SHOW}, - {"DATABASES", TK_DATABASES}, - {"MNODES", TK_MNODES}, - {"DNODES", TK_DNODES}, - {"USERS", TK_USERS}, - {"MODULES", TK_MODULES}, - {"QUERIES", TK_QUERIES}, - {"CONNECTIONS", TK_CONNECTIONS}, - {"STREAMS", TK_STREAMS}, - {"CONFIGS", TK_CONFIGS}, - {"SCORES", TK_SCORES}, - {"GRANTS", TK_GRANTS}, - {"DOT", TK_DOT}, - {"TABLES", TK_TABLES}, - {"STABLES", TK_STABLES}, - {"VGROUPS", TK_VGROUPS}, - {"DROP", TK_DROP}, - {"TABLE", TK_TABLE}, - {"DATABASE", TK_DATABASE}, - {"USER", TK_USER}, - {"USE", TK_USE}, - {"DESCRIBE", TK_DESCRIBE}, - {"ALTER", TK_ALTER}, - {"PASS", TK_PASS}, - {"PRIVILEGE", TK_PRIVILEGE}, - {"DNODE", TK_DNODE}, - {"IP", TK_IP}, - {"LOCAL", TK_LOCAL}, - {"IF", TK_IF}, - {"EXISTS", TK_EXISTS}, - {"CREATE", TK_CREATE}, - {"KEEP", TK_KEEP}, - {"REPLICA", TK_REPLICA}, - {"DAYS", TK_DAYS}, - {"ROWS", TK_ROWS}, - {"CACHE", TK_CACHE}, - {"ABLOCKS", TK_ABLOCKS}, - {"TBLOCKS", TK_TBLOCKS}, - {"CTIME", TK_CTIME}, - {"CLOG", TK_CLOG}, - {"COMP", TK_COMP}, - {"PRECISION", TK_PRECISION}, - {"LP", TK_LP}, - {"RP", TK_RP}, - {"TAGS", TK_TAGS}, - {"USING", TK_USING}, - {"AS", TK_AS}, - {"COMMA", TK_COMMA}, - {"NULL", TK_NULL}, - {"SELECT", TK_SELECT}, - {"FROM", TK_FROM}, - {"VARIABLE", TK_VARIABLE}, - {"INTERVAL", TK_INTERVAL}, - {"FILL", TK_FILL}, - {"SLIDING", TK_SLIDING}, - {"ORDER", TK_ORDER}, - {"BY", TK_BY}, - {"ASC", TK_ASC}, - {"DESC", TK_DESC}, - {"GROUP", TK_GROUP}, - {"HAVING", TK_HAVING}, - {"LIMIT", TK_LIMIT}, - {"OFFSET", TK_OFFSET}, - {"SLIMIT", TK_SLIMIT}, - {"SOFFSET", TK_SOFFSET}, - {"WHERE", TK_WHERE}, - {"NOW", TK_NOW}, - {"INSERT", TK_INSERT}, - {"INTO", TK_INTO}, - {"VALUES", TK_VALUES}, - {"RESET", TK_RESET}, - {"QUERY", TK_QUERY}, - {"ADD", TK_ADD}, - {"COLUMN", TK_COLUMN}, - {"TAG", TK_TAG}, - {"CHANGE", TK_CHANGE}, - {"SET", TK_SET}, - {"KILL", TK_KILL}, - {"CONNECTION", TK_CONNECTION}, - {"COLON", TK_COLON}, - {"STREAM", TK_STREAM}, - {"ABORT", TK_ABORT}, - {"AFTER", TK_AFTER}, - {"ATTACH", TK_ATTACH}, - {"BEFORE", TK_BEFORE}, - {"BEGIN", TK_BEGIN}, - {"CASCADE", TK_CASCADE}, - {"CLUSTER", TK_CLUSTER}, - {"CONFLICT", TK_CONFLICT}, - {"COPY", TK_COPY}, - {"DEFERRED", TK_DEFERRED}, - {"DELIMITERS", TK_DELIMITERS}, - {"DETACH", TK_DETACH}, - {"EACH", TK_EACH}, - {"END", TK_END}, - {"EXPLAIN", TK_EXPLAIN}, - {"FAIL", TK_FAIL}, - {"FOR", TK_FOR}, - {"IGNORE", TK_IGNORE}, - {"IMMEDIATE", TK_IMMEDIATE}, - {"INITIALLY", TK_INITIALLY}, - {"INSTEAD", TK_INSTEAD}, - {"MATCH", TK_MATCH}, - {"KEY", TK_KEY}, - {"OF", TK_OF}, - {"RAISE", TK_RAISE}, - {"REPLACE", TK_REPLACE}, - {"RESTRICT", TK_RESTRICT}, - {"ROW", TK_ROW}, - {"STATEMENT", TK_STATEMENT}, - {"TRIGGER", TK_TRIGGER}, - {"VIEW", TK_VIEW}, - {"ALL", TK_ALL}, - {"COUNT", TK_COUNT}, - {"SUM", TK_SUM}, - {"AVG", TK_AVG}, - {"MIN", TK_MIN}, - {"MAX", TK_MAX}, - {"FIRST", TK_FIRST}, - {"LAST", TK_LAST}, - {"TOP", TK_TOP}, - {"BOTTOM", TK_BOTTOM}, - {"STDDEV", TK_STDDEV}, - {"PERCENTILE", TK_PERCENTILE}, - {"APERCENTILE", TK_APERCENTILE}, - {"LEASTSQUARES",TK_LEASTSQUARES}, - {"HISTOGRAM", TK_HISTOGRAM}, - {"DIFF", TK_DIFF}, - {"SPREAD", TK_SPREAD}, - {"WAVG", TK_WAVG}, - {"INTERP", TK_INTERP}, - {"LAST_ROW", TK_LAST_ROW}, - {"SEMI", TK_SEMI,}, - {"NONE", TK_NONE,}, - {"PREV", TK_PREV,}, - {"LINEAR", TK_LINEAR,}, - {"IMPORT", TK_IMPORT}, - {"METRIC", TK_METRIC}, - {"TBNAME", TK_TBNAME}, - {"JOIN", TK_JOIN}, - {"METRICS", TK_METRICS}, - {"STABLE", TK_STABLE} + {"ID", TK_ID}, + {"BOOL", TK_BOOL}, + {"TINYINT", TK_TINYINT}, + {"SMALLINT", TK_SMALLINT}, + {"INTEGER", TK_INTEGER}, + {"INT", TK_INTEGER}, + {"BIGINT", TK_BIGINT}, + {"FLOAT", TK_FLOAT}, + {"DOUBLE", TK_DOUBLE}, + {"STRING", TK_STRING}, + {"TIMESTAMP", TK_TIMESTAMP}, + {"BINARY", TK_BINARY}, + {"NCHAR", TK_NCHAR}, + {"OR", TK_OR}, + {"AND", TK_AND}, + {"NOT", TK_NOT}, + {"EQ", TK_EQ}, + {"NE", TK_NE}, + {"ISNULL", TK_ISNULL}, + {"NOTNULL", TK_NOTNULL}, + {"IS", TK_IS}, + {"LIKE", TK_LIKE}, + {"GLOB", TK_GLOB}, + {"BETWEEN", TK_BETWEEN}, + {"IN", TK_IN}, + {"GT", TK_GT}, + {"GE", TK_GE}, + {"LT", TK_LT}, + {"LE", TK_LE}, + {"BITAND", TK_BITAND}, + {"BITOR", TK_BITOR}, + {"LSHIFT", TK_LSHIFT}, + {"RSHIFT", TK_RSHIFT}, + {"PLUS", TK_PLUS}, + {"MINUS", TK_MINUS}, + {"DIVIDE", TK_DIVIDE}, + {"TIMES", TK_TIMES}, + {"STAR", TK_STAR}, + {"SLASH", TK_SLASH}, + {"REM ", TK_REM}, + {"CONCAT", TK_CONCAT}, + {"UMINUS", TK_UMINUS}, + {"UPLUS", TK_UPLUS}, + {"BITNOT", TK_BITNOT}, + {"SHOW", TK_SHOW}, + {"DATABASES", TK_DATABASES}, + {"MNODES", TK_MNODES}, + {"DNODES", TK_DNODES}, + {"ACCOUNTS", TK_ACCOUNTS}, + {"USERS", TK_USERS}, + {"MODULES", TK_MODULES}, + {"QUERIES", TK_QUERIES}, + {"CONNECTIONS", TK_CONNECTIONS}, + {"STREAMS", TK_STREAMS}, + {"CONFIGS", TK_CONFIGS}, + {"SCORES", TK_SCORES}, + {"GRANTS", TK_GRANTS}, + {"DOT", TK_DOT}, + {"TABLES", TK_TABLES}, + {"STABLES", TK_STABLES}, + {"VGROUPS", TK_VGROUPS}, + {"DROP", TK_DROP}, + {"TABLE", TK_TABLE}, + {"DATABASE", TK_DATABASE}, + {"DNODE", TK_DNODE}, + {"IP", TK_IP}, + {"USER", TK_USER}, + {"ACCOUNT", TK_ACCOUNT}, + {"USE", TK_USE}, + {"DESCRIBE", TK_DESCRIBE}, + {"ALTER", TK_ALTER}, + {"PASS", TK_PASS}, + {"PRIVILEGE", TK_PRIVILEGE}, + {"LOCAL", TK_LOCAL}, + {"IF", TK_IF}, + {"EXISTS", TK_EXISTS}, + {"CREATE", TK_CREATE}, + {"PPS", TK_PPS}, + {"TSERIES", TK_TSERIES}, + {"DBS", TK_DBS}, + {"STORAGE", TK_STORAGE}, + {"QTIME", TK_QTIME}, + {"CONNS", TK_CONNS}, + {"STATE", TK_STATE}, + {"KEEP", TK_KEEP}, + {"REPLICA", TK_REPLICA}, + {"DAYS", TK_DAYS}, + {"ROWS", TK_ROWS}, + {"CACHE", TK_CACHE}, + {"ABLOCKS", TK_ABLOCKS}, + {"TBLOCKS", TK_TBLOCKS}, + {"CTIME", TK_CTIME}, + {"CLOG", TK_CLOG}, + {"COMP", TK_COMP}, + {"PRECISION", TK_PRECISION}, + {"LP", TK_LP}, + {"RP", TK_RP}, + {"TAGS", TK_TAGS}, + {"USING", TK_USING}, + {"AS", TK_AS}, + {"COMMA", TK_COMMA}, + {"NULL", TK_NULL}, + {"SELECT", TK_SELECT}, + {"FROM", TK_FROM}, + {"VARIABLE", TK_VARIABLE}, + {"INTERVAL", TK_INTERVAL}, + {"FILL", TK_FILL}, + {"SLIDING", TK_SLIDING}, + {"ORDER", TK_ORDER}, + {"BY", TK_BY}, + {"ASC", TK_ASC}, + {"DESC", TK_DESC}, + {"GROUP", TK_GROUP}, + {"HAVING", TK_HAVING}, + {"LIMIT", TK_LIMIT}, + {"OFFSET", TK_OFFSET}, + {"SLIMIT", TK_SLIMIT}, + {"SOFFSET", TK_SOFFSET}, + {"WHERE", TK_WHERE}, + {"NOW", TK_NOW}, + {"INSERT", TK_INSERT}, + {"INTO", TK_INTO}, + {"VALUES", TK_VALUES}, + {"RESET", TK_RESET}, + {"QUERY", TK_QUERY}, + {"ADD", TK_ADD}, + {"COLUMN", TK_COLUMN}, + {"TAG", TK_TAG}, + {"CHANGE", TK_CHANGE}, + {"SET", TK_SET}, + {"KILL", TK_KILL}, + {"CONNECTION", TK_CONNECTION}, + {"COLON", TK_COLON}, + {"STREAM", TK_STREAM}, + {"ABORT", TK_ABORT}, + {"AFTER", TK_AFTER}, + {"ATTACH", TK_ATTACH}, + {"BEFORE", TK_BEFORE}, + {"BEGIN", TK_BEGIN}, + {"CASCADE", TK_CASCADE}, + {"CLUSTER", TK_CLUSTER}, + {"CONFLICT", TK_CONFLICT}, + {"COPY", TK_COPY}, + {"DEFERRED", TK_DEFERRED}, + {"DELIMITERS", TK_DELIMITERS}, + {"DETACH", TK_DETACH}, + {"EACH", TK_EACH}, + {"END", TK_END}, + {"EXPLAIN", TK_EXPLAIN}, + {"FAIL", TK_FAIL}, + {"FOR", TK_FOR}, + {"IGNORE", TK_IGNORE}, + {"IMMEDIATE", TK_IMMEDIATE}, + {"INITIALLY", TK_INITIALLY}, + {"INSTEAD", TK_INSTEAD}, + {"MATCH", TK_MATCH}, + {"KEY", TK_KEY}, + {"OF", TK_OF}, + {"RAISE", TK_RAISE}, + {"REPLACE", TK_REPLACE}, + {"RESTRICT", TK_RESTRICT}, + {"ROW", TK_ROW}, + {"STATEMENT", TK_STATEMENT}, + {"TRIGGER", TK_TRIGGER}, + {"VIEW", TK_VIEW}, + {"ALL", TK_ALL}, + {"COUNT", TK_COUNT}, + {"SUM", TK_SUM}, + {"AVG", TK_AVG}, + {"MIN", TK_MIN}, + {"MAX", TK_MAX}, + {"FIRST", TK_FIRST}, + {"LAST", TK_LAST}, + {"TOP", TK_TOP}, + {"BOTTOM", TK_BOTTOM}, + {"STDDEV", TK_STDDEV}, + {"PERCENTILE", TK_PERCENTILE}, + {"APERCENTILE", TK_APERCENTILE}, + {"LEASTSQUARES", TK_LEASTSQUARES}, + {"HISTOGRAM", TK_HISTOGRAM}, + {"DIFF", TK_DIFF}, + {"SPREAD", TK_SPREAD}, + {"TWA", TK_TWA}, + {"INTERP", TK_INTERP}, + {"LAST_ROW", TK_LAST_ROW}, + {"SEMI", TK_SEMI}, + {"NONE", TK_NONE}, + {"PREV", TK_PREV}, + {"LINEAR", TK_LINEAR}, + {"IMPORT", TK_IMPORT}, + {"METRIC", TK_METRIC}, + {"TBNAME", TK_TBNAME}, + {"JOIN", TK_JOIN}, + {"METRICS", TK_METRICS}, + {"STABLE", TK_STABLE}, + {"FILE", TK_FILE}, }; /* This is the hash table */ @@ -234,11 +245,11 @@ static const char isIdChar[] = { }; static void* KeywordHashTable = NULL; -int tSQLKeywordCode(const char* z, int n) { +int tSQLKeywordCode(const char* z, int n) { int i; static char needInit = 1; if (needInit) { - /* Initialize the keyword hash table */ + // Initialize the keyword hash table pthread_mutex_lock(&mutex); // double check @@ -400,10 +411,14 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { *tokenType = TK_BITNOT; return 1; } + case '?': { + *tokenType = TK_QUESTION; + return 1; + } case '\'': case '"': { - int delim = z[0]; - bool strEnd = false; + int delim = z[0]; + bool strEnd = false; for (i = 1; z[i]; i++) { if (z[i] == delim) { if (z[i + 1] == delim) { @@ -416,18 +431,64 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { } if (z[i]) i++; - if (strEnd){ - *tokenType = TK_STRING; - return i; - } + if (strEnd) { + *tokenType = TK_STRING; + return i; + } - break; + break; } case '.': { - *tokenType = TK_DOT; - return 1; + /* + * handle the the float number with out integer part + * .123 + * .123e4 + */ + if (isdigit(z[1])) { + for (i = 2; isdigit(z[i]); i++) { + } + + if ((z[i] == 'e' || z[i] == 'E') && + (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) { + i += 2; + while (isdigit(z[i])) { + i++; + } + } + + *tokenType = TK_FLOAT; + return i; + } else { + *tokenType = TK_DOT; + return 1; + } + } + + case '0': { + char next = z[1]; + + if (next == 'b') { // bin number + *tokenType = TK_BIN; + for (i = 2; (z[i] == '0' || z[i] == '1'); ++i) { + } + + if (i == 2) { + break; + } + + return i; + } else if (next == 'x') { //hex number + *tokenType = TK_HEX; + for (i = 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) { + } + + if (i == 2) { + break; + } + + return i; + } } - case '0': case '1': case '2': case '3': @@ -443,8 +504,8 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { /* here is the 1a/2s/3m/9y */ if ((z[i] == 'a' || z[i] == 's' || z[i] == 'm' || z[i] == 'h' || z[i] == 'd' || z[i] == 'n' || z[i] == 'y' || - z[i] == 'w' || z[i] == 'A' || z[i] == 'S' || z[i] == 'M' || z[i] == 'H' || z[i] == 'D' || z[i] == 'N' || - z[i] == 'Y' || z[i] == 'W') && + z[i] == 'w' || z[i] == 'A' || z[i] == 'S' || z[i] == 'M' || z[i] == 'H' || z[i] == 'D' || z[i] == 'N' || + z[i] == 'Y' || z[i] == 'W') && (isIdChar[z[i + 1]] == 0)) { *tokenType = TK_VARIABLE; i += 1; @@ -509,38 +570,195 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { return 0; } -void tStrGetToken(char* str, int32_t* i, SSQLToken* t0, bool isPrevOptr) { +SSQLToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t* ignoreTokenTypes) { + SSQLToken t0 = {0}; + // here we reach the end of sql string, null-terminated string if (str[*i] == 0) { - t0->n = 0; - return; + t0.n = 0; + return t0; } - t0->n = tSQLGetToken(&str[*i], &t0->type); + // IGNORE TK_SPACE, TK_COMMA, and specified tokens + while (1) { + *i += t0.n; + + bool hasComma = false; + while ((str[*i] == ' ' || str[*i] == '\n' || str[*i] == '\r' || str[*i] == '\t' || str[*i] == '\f') + || str[*i] == ',') { + if (str[*i] == ',') { + if (false == hasComma) { + hasComma = true; + } else { // comma only allowed once + t0.n = 0; + return t0; + } + } + (*i)++; + } - /* IGNORE all space between valid tokens */ - while (t0->type == TK_SPACE) { - *i += t0->n; - t0->n = tSQLGetToken(&str[*i], &t0->type); + t0.n = tSQLGetToken(&str[*i], &t0.type); + + bool ignoreFlag = false; + for (uint32_t k = 0; k < numOfIgnoreToken; k++) { + if (t0.type == ignoreTokenTypes[k]) { + ignoreFlag = true; + break; + } + } + + if (!ignoreFlag) { + break; + } } - /* support parse the -/+number format */ - if ((isPrevOptr) && (t0->type == TK_MINUS || t0->type == TK_PLUS)) { - uint32_t type = 0; - int32_t len = tSQLGetToken(&str[*i + t0->n], &type); - if (type == TK_INTEGER || type == TK_FLOAT) { - t0->type = type; - t0->n += len; + if (t0.type == TK_SEMI) { + t0.n = 0; + return t0; + } + + uint32_t type = 0; + int32_t len; + + // support parse the 'db.tbl' format, notes: There should be no space on either side of the dot! + if ('.' == str[*i + t0.n]) { + len = tSQLGetToken(&str[*i + t0.n + 1], &type); + + // only id and string are valid + if ((TK_STRING != t0.type) && (TK_ID != t0.type)) { + t0.type = TK_ILLEGAL; + t0.n = 0; + + return t0; + } + + t0.n += len + 1; + + } else { + // support parse the -/+number format + if ((isPrevOptr) && (t0.type == TK_MINUS || t0.type == TK_PLUS)) { + len = tSQLGetToken(&str[*i + t0.n], &type); + if (type == TK_INTEGER || type == TK_FLOAT) { + t0.type = type; + t0.n += len; + } } } - t0->z = str + (*i); - *i += t0->n; + t0.z = str + (*i); + *i += t0.n; + + return t0; } -bool isKeyWord(const char* z, int32_t len) { - int32_t tokenType = tSQLKeywordCode((char*)z, len); - return (tokenType != TK_ID); +FORCE_INLINE bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); } + +FORCE_INLINE bool isNumber(const SSQLToken* pToken) { + return (pToken->type == TK_INTEGER || pToken->type == TK_FLOAT || pToken->type == TK_HEX || pToken->type == TK_BIN); } -bool isNumber(const SSQLToken* pToken) { return (pToken->type == TK_INTEGER || pToken->type == TK_FLOAT); } +int32_t isValidNumber(const SSQLToken* pToken) { + const char* z = pToken->z; + int32_t type = TK_ILLEGAL; + + int32_t i = 0; + for(; i < pToken->n; ++i) { + switch (z[i]) { + case '+': + case '-': { + break; + } + case '.': { + /* + * handle the the float number with out integer part + * .123 + * .123e4 + */ + if (!isdigit(z[i+1])) { + return TK_ILLEGAL; + } + + for (i += 2; isdigit(z[i]); i++) { + } + + if ((z[i] == 'e' || z[i] == 'E') && + (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) { + i += 2; + while (isdigit(z[i])) { + i++; + } + } + + type = TK_FLOAT; + goto _end; + } + + case '0': { + char next = z[i + 1]; + if (next == 'b') { // bin number + type = TK_BIN; + for (i += 2; (z[i] == '0' || z[i] == '1'); ++i) { + } + + goto _end; + } else if (next == 'x') { //hex number + type = TK_HEX; + for (i += 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) { + } + + goto _end; + } + } + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + type = TK_INTEGER; + for (; isdigit(z[i]); i++) { + } + + int32_t seg = 0; + while (z[i] == '.' && isdigit(z[i + 1])) { + i += 2; + + while (isdigit(z[i])) { + i++; + } + + seg++; + type = TK_FLOAT; + } + + if (seg > 1) { + return TK_ILLEGAL; + } + + if ((z[i] == 'e' || z[i] == 'E') && + (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) { + i += 2; + while (isdigit(z[i])) { + i++; + } + + type = TK_FLOAT; + } + + goto _end; + } + default: + return TK_ILLEGAL; + } + } + + _end: + if (i < pToken->n) { + return TK_ILLEGAL; + } else { + return type; + } +} \ No newline at end of file diff --git a/src/util/src/ttypes.c b/src/util/src/ttypes.c index eb6887d30f..3b36b1b31c 100644 --- a/src/util/src/ttypes.c +++ b/src/util/src/ttypes.c @@ -73,9 +73,9 @@ bool isValidDataType(int32_t type, int32_t length) { } // todo support scientific expression number and oct number -void tVariantCreate(tVariant *pVar, SSQLToken *token) { tVariantCreateN(pVar, token->z, token->n, token->type); } +void tVariantCreate(tVariant *pVar, SSQLToken *token) { tVariantCreateFromString(pVar, token->z, token->n, token->type); } -void tVariantCreateN(tVariant *pVar, char *pz, uint32_t len, uint32_t type) { +void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: { int32_t k = strncasecmp(pz, "true", 4); @@ -98,11 +98,7 @@ void tVariantCreateN(tVariant *pVar, char *pz, uint32_t len, uint32_t type) { pVar->dKey = strtod(pz, NULL); break; case TSDB_DATA_TYPE_BINARY: { - pVar->pz = malloc(len + 1); - strncpy(pVar->pz, pz, len); - pVar->nLen = len; - pVar->pz[len] = 0; - + pVar->pz = strndup(pz, len); pVar->nLen = strdequote(pVar->pz); break; } @@ -122,7 +118,7 @@ void tVariantCreateN(tVariant *pVar, char *pz, uint32_t len, uint32_t type) { * @param len * @param type */ -void tVariantCreateB(tVariant *pVar, char *pz, uint32_t len, uint32_t type) { +void tVariantCreateFromBinary(tVariant *pVar, char *pz, uint32_t len, uint32_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: { @@ -150,8 +146,7 @@ void tVariantCreateB(tVariant *pVar, char *pz, uint32_t len, uint32_t type) { pVar->dKey = GET_FLOAT_VAL(pz); break; } - case TSDB_DATA_TYPE_NCHAR: { - /* here we get the nchar length from raw binary bits length */ + case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length pVar->nLen = len / TSDB_NCHAR_SIZE; pVar->wpz = malloc((pVar->nLen + 1) * TSDB_NCHAR_SIZE); @@ -161,16 +156,12 @@ void tVariantCreateB(tVariant *pVar, char *pz, uint32_t len, uint32_t type) { break; } case TSDB_DATA_TYPE_BINARY: { - pVar->pz = malloc(len + 1); - strncpy(pVar->pz, pz, len); - - pVar->nLen = len; - pVar->pz[len] = 0; - + pVar->pz = strndup(pz, len); pVar->nLen = strdequote(pVar->pz); break; } + default: pVar->i64Key = GET_INT32_VAL(pVar); } @@ -242,9 +233,13 @@ int32_t tVariantToString(tVariant *pVar, char *dst) { } static int32_t doConvertToInteger(tVariant *pVariant, char *pDest, int32_t type, bool releaseVariantPtr) { + if (pVariant->nType == TSDB_DATA_TYPE_NULL) { + setNull(pDest, type, tDataTypeDesc[type].nSize); + return 0; + } + if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { *((int64_t *)pDest) = pVariant->i64Key; - } else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) { if ((pVariant->dKey < INT64_MIN) || (pVariant->dKey > INT64_MAX)) { return -1; @@ -290,8 +285,11 @@ static int32_t doConvertToInteger(tVariant *pVariant, char *pDest, int32_t type, } *((int64_t *)pDest) = val; - } else if (strncasecmp(TSDB_DATA_NULL_STR_L, pVariant->pz, pVariant->nLen) == 0 && - strlen(TSDB_DATA_NULL_STR_L) == pVariant->nLen) { + } else if (token.type == TK_NULL) { + if (releaseVariantPtr) { + free(pVariant->pz); + pVariant->nLen = 0; + } setNull(pDest, type, tDataTypeDesc[type].nSize); } else { return -1; @@ -324,6 +322,12 @@ static int32_t doConvertToInteger(tVariant *pVariant, char *pDest, int32_t type, } *((int64_t *)pDest) = (int64_t)v; + } else if (token.type == TK_NULL) { + if (releaseVariantPtr) { + free(pVariant->pz); + pVariant->nLen = 0; + } + setNull(pDest, type, tDataTypeDesc[type].nSize); } else { int64_t val = wcstoll(pVariant->wpz, &endPtr, 10); if (releaseVariantPtr) { @@ -395,12 +399,15 @@ static int32_t toBinary(tVariant *pVariant, char **pDest, int32_t *pDestSize) { sprintf(pBuf == NULL ? *pDest : pBuf, "%lf", pVariant->dKey); } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL) { sprintf(pBuf == NULL ? *pDest : pBuf, "%s", (pVariant->i64Key == TSDB_TRUE) ? "TRUE" : "FALSE"); + } else if (pVariant->nType == 0) { // null data + setNull(pBuf == NULL ? *pDest : pBuf, TSDB_DATA_TYPE_BINARY, 0); } } if (pBuf != NULL) { *pDest = pBuf; } + *pDestSize = strlen(*pDest); return 0; } @@ -426,12 +433,19 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { wchar_t *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE); + // free the binary buffer in the first place if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { free(pVariant->wpz); } - pVariant->wpz = pWStr; + pVariant->wpz = pWStr; *pDestSize = wcslen(pVariant->wpz); + + // shrink the allocate memory, no need to check here. + char* tmp = realloc(pVariant->wpz, (*pDestSize + 1)*TSDB_NCHAR_SIZE); + assert(tmp != NULL); + + pVariant->wpz = tmp; } else { taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE); } @@ -439,17 +453,127 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { return 0; } -static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result, int32_t type, int64_t lowBnd, - int64_t upperBnd) { - if (doConvertToInteger(pVariant, (char *)result, type, false) != 0) { +static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *value) { + SSQLToken stoken = {.z = pStr, .n = len}; + + if (TK_ILLEGAL == isValidNumber(&stoken)) { return -1; } - if (isNull((char *)result, type)) { + *value = strtod(pStr, NULL); + + return 0; +} + +static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result, int32_t type, int64_t lowBnd, + int64_t upperBnd, bool releaseVariantPtr) { + if (pVariant->nType == TSDB_DATA_TYPE_NULL) { + setNull((char *)result, type, tDataTypeDesc[type].nSize); return 0; } - if (*result < lowBnd || *result > upperBnd) { + if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { + *result = pVariant->i64Key; + } else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) { + *result = (int64_t)pVariant->dKey; + } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { + errno = 0; + char *endPtr = NULL; + + SSQLToken token = {0}; + token.n = tSQLGetToken(pVariant->pz, &token.type); + + if (token.type == TK_MINUS || token.type == TK_PLUS) { + token.n = tSQLGetToken(pVariant->pz + token.n, &token.type); + } + + if (token.type == TK_NULL) { + if (releaseVariantPtr) { + free(pVariant->pz); + pVariant->nLen = 0; + } + setNull((char *)result, type, tDataTypeDesc[type].nSize); + return 0; + } + + SSQLToken sToken = {.z = pVariant->pz, .n = pVariant->nLen}; + if (TK_ILLEGAL == isValidNumber(&sToken)) { + return -1; + } + + if (token.type == TK_FLOAT) { + double v = strtod(pVariant->pz, &endPtr); + if (releaseVariantPtr) { + free(pVariant->pz); + pVariant->nLen = 0; + } + + if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { + return -1; + } + + *result = (int64_t)v; + } else if (token.type == TK_INTEGER) { + int64_t val = strtoll(pVariant->pz, &endPtr, 10); + if (releaseVariantPtr) { + free(pVariant->pz); + pVariant->nLen = 0; + } + + if (errno == ERANGE) { + return -1; // data overflow + } + + *result = val; + } else { + return -1; + } + } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { + errno = 0; + wchar_t *endPtr = NULL; + + SSQLToken token = {0}; + token.n = tSQLGetToken(pVariant->pz, &token.type); + + if (token.type == TK_MINUS || token.type == TK_PLUS) { + token.n = tSQLGetToken(pVariant->pz + token.n, &token.type); + } + + if (token.type == TK_FLOAT) { + double v = wcstod(pVariant->wpz, &endPtr); + if (releaseVariantPtr) { + free(pVariant->pz); + pVariant->nLen = 0; + } + + if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { + return -1; + } + + *result = (int64_t)v; + } else if (token.type == TK_NULL) { + if (releaseVariantPtr) { + free(pVariant->pz); + pVariant->nLen = 0; + } + setNull((char *)result, type, tDataTypeDesc[type].nSize); + return 0; + } else { + int64_t val = wcstoll(pVariant->wpz, &endPtr, 10); + if (releaseVariantPtr) { + free(pVariant->pz); + pVariant->nLen = 0; + } + + if (errno == ERANGE) { + return -1; // data overflow + } + + *result = val; + } + } + + if ((*result <= lowBnd) || (*result > upperBnd)) { return -1; } @@ -459,7 +583,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result static int32_t convertToBool(tVariant *pVariant, int64_t *pDest) { if (pVariant->nType == TSDB_DATA_TYPE_BOOL) { *pDest = pVariant->i64Key; // in order to be compatible to null of bool - } else if (pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { + } else if (pVariant->nType >= TSDB_DATA_TYPE_TINYINT && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { *pDest = ((pVariant->i64Key != 0) ? TSDB_TRUE : TSDB_FALSE); } else if (pVariant->nType == TSDB_DATA_TYPE_FLOAT || pVariant->nType == TSDB_DATA_TYPE_DOUBLE) { *pDest = ((pVariant->dKey != 0) ? TSDB_TRUE : TSDB_FALSE); @@ -475,8 +599,9 @@ static int32_t convertToBool(tVariant *pVariant, int64_t *pDest) { if ((ret = wcsconvertToBoolImpl(pVariant->wpz, pVariant->nLen)) < 0) { return ret; } - *pDest = ret; + } else if (pVariant->nType == TSDB_DATA_TYPE_NULL) { + *pDest = TSDB_DATA_BOOL_NULL; } assert(*pDest == TSDB_TRUE || *pDest == TSDB_FALSE || *pDest == TSDB_DATA_BOOL_NULL); @@ -490,15 +615,14 @@ static int32_t convertToBool(tVariant *pVariant, int64_t *pDest) { * todo handle the return value */ int32_t tVariantDump(tVariant *pVariant, char *payload, char type) { - if (pVariant == NULL || !isValidDataType(pVariant->nType, pVariant->nLen)) { // value is not set + if (pVariant == NULL || (pVariant->nType != 0 && !isValidDataType(pVariant->nType, pVariant->nLen))) { return -1; } - char *endPtr = NULL; errno = 0; // reset global error code switch (type) { - case TSDB_DATA_TYPE_BOOL: { // bool + case TSDB_DATA_TYPE_BOOL: { int64_t dst = 0; if (convertToBool(pVariant, &dst) < 0) { return -1; @@ -506,9 +630,10 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, char type) { *(int8_t *)payload = (int8_t)dst; break; } + case TSDB_DATA_TYPE_TINYINT: { int64_t result = 0; - if (convertToInteger(pVariant, &result, type, INT8_MIN, INT8_MAX) < 0) { + if (convertToInteger(pVariant, &result, type, INT8_MIN, INT8_MAX, false) < 0) { return -1; } @@ -518,7 +643,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, char type) { case TSDB_DATA_TYPE_SMALLINT: { int64_t result = 0; - if (convertToInteger(pVariant, &result, type, INT16_MIN, INT16_MAX) < 0) { + if (convertToInteger(pVariant, &result, type, INT16_MIN, INT16_MAX, false) < 0) { return -1; } @@ -528,93 +653,164 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, char type) { case TSDB_DATA_TYPE_INT: { int64_t result = 0; - if (convertToInteger(pVariant, &result, type, INT32_MIN, INT32_MAX) < 0) { + if (convertToInteger(pVariant, &result, type, INT32_MIN, INT32_MAX, false) < 0) { return -1; } *((int32_t *)payload) = (int32_t)result; break; } + case TSDB_DATA_TYPE_BIGINT: { int64_t result = 0; - if (doConvertToInteger(pVariant, (char *)&result, type, false) != 0) { + if (convertToInteger(pVariant, &result, type, INT64_MIN, INT64_MAX, false) < 0) { return -1; } *((int64_t *)payload) = (int64_t)result; break; } + case TSDB_DATA_TYPE_FLOAT: { if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { if (strncasecmp(TSDB_DATA_NULL_STR_L, pVariant->pz, pVariant->nLen) == 0 && strlen(TSDB_DATA_NULL_STR_L) == pVariant->nLen) { - setNull(payload, type, tDataTypeDesc[type].nSize); + *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; + return 0; } else { - *((float *)payload) = (float)strtod(pVariant->pz, &endPtr); - if (errno == ERANGE && *(float *)payload == -1) { + double value; + int32_t ret; + ret = convertToDouble(pVariant->pz, pVariant->nLen, &value); + if ((errno == ERANGE && (float)value == -1) || (ret != 0)) { return -1; } - if (isinf(*(float *)payload) || isnan(*(float *)payload)) { - return -1; - } +#ifdef _TD_ARM_32_ + //memcpy(&payload, &value, sizeof(float)); + float fv = (float)value; + SET_FLOAT_VAL_ALIGN(payload, &fv); +#else + *((float *)payload) = (float)value; +#endif } } else if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { +#ifdef _TD_ARM_32_ + //memcpy(&payload, &pVariant->i64Key, sizeof(float)); + float fv = (float)pVariant->i64Key; + SET_FLOAT_VAL_ALIGN(payload, &fv); +#else *((float *)payload) = pVariant->i64Key; +#endif } else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) { +#ifdef _TD_ARM_32_ + //memcpy(&payload, &pVariant->dKey, sizeof(float)); + float fv = (float)pVariant->dKey; + SET_FLOAT_VAL_ALIGN(payload, &fv); +#else *((float *)payload) = (float)pVariant->dKey; +#endif + } else if (pVariant->nType == TSDB_DATA_TYPE_NULL) { + *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; + return 0; + } - if (isinf(pVariant->dKey) || isnan(pVariant->dKey) || pVariant->dKey > FLT_MAX || pVariant->dKey < -FLT_MAX) { - return -1; - } +#ifdef _TD_ARM_32_ + float fv = GET_FLOAT_VAL(payload); + if (isinf(fv) || isnan(fv) || fv > FLT_MAX || fv < -FLT_MAX) { + return -1; + } +#else + if (isinf(*((float *)payload)) || isnan(*((float *)payload)) || *((float *)payload) > FLT_MAX || + *((float *)payload) < -FLT_MAX) { + return -1; } +#endif break; } case TSDB_DATA_TYPE_DOUBLE: { if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { if (strncasecmp(TSDB_DATA_NULL_STR_L, pVariant->pz, pVariant->nLen) == 0 && strlen(TSDB_DATA_NULL_STR_L) == pVariant->nLen) { - setNull(payload, type, tDataTypeDesc[type].nSize); + *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; + return 0; } else { - *((double *)payload) = strtod(pVariant->pz, &endPtr); - if (errno == ERANGE && *(double *)payload == -1) { + double value; + int32_t ret; + ret = convertToDouble(pVariant->pz, pVariant->nLen, &value); + if ((errno == ERANGE && value == -1) || (ret != 0)) { return -1; } - if (isnan(*(double *)payload) || isinf(*(double *)payload)) { - return -1; - } +#ifdef _TD_ARM_32_ + SET_DOUBLE_VAL_ALIGN(payload, &value); +#else + *((double *)payload) = value; +#endif } } else if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { +#ifdef _TD_ARM_32_ + double dv = (double)(pVariant->i64Key); + SET_DOUBLE_VAL_ALIGN(payload, &dv); +#else *((double *)payload) = pVariant->i64Key; +#endif } else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) { +#ifdef _TD_ARM_32_ + double dv = (double)(pVariant->dKey); + SET_DOUBLE_VAL_ALIGN(payload, &dv); +#else *((double *)payload) = pVariant->dKey; - if (isnan(pVariant->dKey) || isinf(pVariant->dKey)) { - return -1; - } +#endif + } else if (pVariant->nType == TSDB_DATA_TYPE_NULL) { + *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; + return 0; } +#ifdef _TD_ARM_32_ + double dv = GET_DOUBLE_VAL(payload); + if (isinf(dv) || isnan(dv) || dv > DBL_MAX || dv < -DBL_MAX) { + return -1; + } +#else + if (isinf(*((double *)payload)) || isnan(*((double *)payload)) || *((double *)payload) > DBL_MAX || + *((double *)payload) < -DBL_MAX) { + return -1; + } +#endif break; } case TSDB_DATA_TYPE_BINARY: { - if (pVariant->nType != TSDB_DATA_TYPE_BINARY) { - toBinary(pVariant, &payload, &pVariant->nLen); + if (pVariant->nType == TSDB_DATA_TYPE_NULL) { + *payload = TSDB_DATA_BINARY_NULL; } else { - strncpy(payload, pVariant->pz, pVariant->nLen); + if (pVariant->nType != TSDB_DATA_TYPE_BINARY) { + toBinary(pVariant, &payload, &pVariant->nLen); + } else { + strncpy(payload, pVariant->pz, pVariant->nLen); + } } break; } case TSDB_DATA_TYPE_TIMESTAMP: { - *((int64_t *)payload) = pVariant->i64Key; + if (pVariant->nType == TSDB_DATA_TYPE_NULL) { + *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; + } else { + *((int64_t *)payload) = pVariant->i64Key; + } break; } case TSDB_DATA_TYPE_NCHAR: { - if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) { - toNchar(pVariant, &payload, &pVariant->nLen); + if (pVariant->nType == TSDB_DATA_TYPE_NULL) { + *(uint32_t *)payload = TSDB_DATA_NCHAR_NULL; } else { - wcsncpy((wchar_t *)payload, pVariant->wpz, pVariant->nLen); + if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) { + toNchar(pVariant, &payload, &pVariant->nLen); + } else { + wcsncpy((wchar_t *)payload, pVariant->wpz, pVariant->nLen); + } } + break; } } @@ -645,7 +841,7 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) { case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: { - doConvertToInteger(pVariant, (char *)&pVariant->i64Key, type, true); + convertToInteger(pVariant, &(pVariant->i64Key), type, INT64_MIN, INT64_MAX, true); pVariant->nType = TSDB_DATA_TYPE_BIGINT; break; } @@ -707,8 +903,8 @@ bool isNull(const char *val, int32_t type) { return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL; case TSDB_DATA_TYPE_INT: return *(uint32_t *)val == TSDB_DATA_INT_NULL; - case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL; case TSDB_DATA_TYPE_FLOAT: return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL; diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index cb93319900..805f27a381 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -500,7 +500,7 @@ bool taosGetVersionNumber(char *versionStr, int *versionNubmer) { return true; } -char *taosIpStr(int ipInt) { +char *taosIpStr(uint32_t ipInt) { static char ipStrArray[3][30]; static int ipStrIndex = 0; @@ -508,3 +508,19 @@ char *taosIpStr(int ipInt) { sprintf(ipStr, "0x%x:%d.%d.%d.%d", ipInt, ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, ipInt >> 24); return ipStr; } + +#ifndef CLUSTER +void taosCleanupTier() {} +#endif + +FORCE_INLINE float taos_align_get_float(char* pBuf) { + float fv = 0; + *(int32_t*)(&fv) = *(int32_t*)pBuf; + return fv; +} + +FORCE_INLINE double taos_align_get_double(char* pBuf) { + double dv = 0; + *(int64_t*)(&dv) = *(int64_t*)pBuf; + return dv; +} diff --git a/src/util/src/version.c b/src/util/src/version.c old mode 100755 new mode 100644 index fa769db684..ea077c951e --- a/src/util/src/version.c +++ b/src/util/src/version.c @@ -1,4 +1,4 @@ -char version[64] = "1.6.3.1"; +char version[64] = "1.6.4.0"; char compatible_version[64] = "1.6.1.0"; -char gitinfo[128] = "c2f65ef028b41213c8c731778eff019814bd0538"; -char buildinfo[512] = "Built by ubuntu at 2019-11-07 11:17"; +char gitinfo[128] = "a5754d4218b3f01d49e62f53190e861120ca0aba"; +char buildinfo[512] = "Built by ubuntu at 2019-11-06 18:24"; diff --git a/tests/comparisonTest/cassandra/q1.txt b/tests/comparisonTest/cassandra/q1.txt index 5a11c4afc8..07dfd7f978 100644 --- a/tests/comparisonTest/cassandra/q1.txt +++ b/tests/comparisonTest/cassandra/q1.txt @@ -1,11 +1,11 @@ -select * from cassandra.test where devgroup=0 allow filtering; -select * from cassandra.test where devgroup=10 allow filtering; -select * from cassandra.test where devgroup=20 allow filtering; -select * from cassandra.test where devgroup=30 allow filtering; -select * from cassandra.test where devgroup=40 allow filtering; -select * from cassandra.test where devgroup=50 allow filtering; -select * from cassandra.test where devgroup=60 allow filtering; -select * from cassandra.test where devgroup=70 allow filtering; -select * from cassandra.test where devgroup=80 allow filtering; -select * from cassandra.test where devgroup=90 allow filtering; - +select * from cassandra.test where devgroup=0 allow filtering; +select * from cassandra.test where devgroup=10 allow filtering; +select * from cassandra.test where devgroup=20 allow filtering; +select * from cassandra.test where devgroup=30 allow filtering; +select * from cassandra.test where devgroup=40 allow filtering; +select * from cassandra.test where devgroup=50 allow filtering; +select * from cassandra.test where devgroup=60 allow filtering; +select * from cassandra.test where devgroup=70 allow filtering; +select * from cassandra.test where devgroup=80 allow filtering; +select * from cassandra.test where devgroup=90 allow filtering; + diff --git a/tests/comparisonTest/cassandra/q2.txt b/tests/comparisonTest/cassandra/q2.txt index 24c867b171..38bba57b97 100644 --- a/tests/comparisonTest/cassandra/q2.txt +++ b/tests/comparisonTest/cassandra/q2.txt @@ -1,50 +1,50 @@ -select count(*) from cassandra.test where devgroup<10 allow filtering; -select count(*) from cassandra.test where devgroup<20 allow filtering; -select count(*) from cassandra.test where devgroup<30 allow filtering; -select count(*) from cassandra.test where devgroup<40 allow filtering; -select count(*) from cassandra.test where devgroup<50 allow filtering; -select count(*) from cassandra.test where devgroup<60 allow filtering; -select count(*) from cassandra.test where devgroup<70 allow filtering; -select count(*) from cassandra.test where devgroup<80 allow filtering; -select count(*) from cassandra.test where devgroup<90 allow filtering; -select count(*) from cassandra.test allow filtering; -select avg(temperature) from cassandra.test where devgroup<10 allow filtering; -select avg(temperature) from cassandra.test where devgroup<20 allow filtering; -select avg(temperature) from cassandra.test where devgroup<30 allow filtering; -select avg(temperature) from cassandra.test where devgroup<40 allow filtering; -select avg(temperature) from cassandra.test where devgroup<50 allow filtering; -select avg(temperature) from cassandra.test where devgroup<60 allow filtering; -select avg(temperature) from cassandra.test where devgroup<70 allow filtering; -select avg(temperature) from cassandra.test where devgroup<80 allow filtering; -select avg(temperature) from cassandra.test where devgroup<90 allow filtering; -select avg(temperature) from cassandra.test allow filtering; -select sum(temperature) from cassandra.test where devgroup<10 allow filtering; -select sum(temperature) from cassandra.test where devgroup<20 allow filtering; -select sum(temperature) from cassandra.test where devgroup<30 allow filtering; -select sum(temperature) from cassandra.test where devgroup<40 allow filtering; -select sum(temperature) from cassandra.test where devgroup<50 allow filtering; -select sum(temperature) from cassandra.test where devgroup<60 allow filtering; -select sum(temperature) from cassandra.test where devgroup<70 allow filtering; -select sum(temperature) from cassandra.test where devgroup<80 allow filtering; -select sum(temperature) from cassandra.test where devgroup<90 allow filtering; -select sum(temperature) from cassandra.test allow filtering; -select max(temperature) from cassandra.test where devgroup<10 allow filtering; -select max(temperature) from cassandra.test where devgroup<20 allow filtering; -select max(temperature) from cassandra.test where devgroup<30 allow filtering; -select max(temperature) from cassandra.test where devgroup<40 allow filtering; -select max(temperature) from cassandra.test where devgroup<50 allow filtering; -select max(temperature) from cassandra.test where devgroup<60 allow filtering; -select max(temperature) from cassandra.test where devgroup<70 allow filtering; -select max(temperature) from cassandra.test where devgroup<80 allow filtering; -select max(temperature) from cassandra.test where devgroup<90 allow filtering; -select max(temperature) from cassandra.test allow filtering; -select min(temperature) from cassandra.test where devgroup<10 allow filtering; -select min(temperature) from cassandra.test where devgroup<20 allow filtering; -select min(temperature) from cassandra.test where devgroup<30 allow filtering; -select min(temperature) from cassandra.test where devgroup<40 allow filtering; -select min(temperature) from cassandra.test where devgroup<50 allow filtering; -select min(temperature) from cassandra.test where devgroup<60 allow filtering; -select min(temperature) from cassandra.test where devgroup<70 allow filtering; -select min(temperature) from cassandra.test where devgroup<80 allow filtering; -select min(temperature) from cassandra.test where devgroup<90 allow filtering; -select min(temperature) from cassandra.test allow filtering; +select count(*) from cassandra.test where devgroup<10 allow filtering; +select count(*) from cassandra.test where devgroup<20 allow filtering; +select count(*) from cassandra.test where devgroup<30 allow filtering; +select count(*) from cassandra.test where devgroup<40 allow filtering; +select count(*) from cassandra.test where devgroup<50 allow filtering; +select count(*) from cassandra.test where devgroup<60 allow filtering; +select count(*) from cassandra.test where devgroup<70 allow filtering; +select count(*) from cassandra.test where devgroup<80 allow filtering; +select count(*) from cassandra.test where devgroup<90 allow filtering; +select count(*) from cassandra.test allow filtering; +select avg(temperature) from cassandra.test where devgroup<10 allow filtering; +select avg(temperature) from cassandra.test where devgroup<20 allow filtering; +select avg(temperature) from cassandra.test where devgroup<30 allow filtering; +select avg(temperature) from cassandra.test where devgroup<40 allow filtering; +select avg(temperature) from cassandra.test where devgroup<50 allow filtering; +select avg(temperature) from cassandra.test where devgroup<60 allow filtering; +select avg(temperature) from cassandra.test where devgroup<70 allow filtering; +select avg(temperature) from cassandra.test where devgroup<80 allow filtering; +select avg(temperature) from cassandra.test where devgroup<90 allow filtering; +select avg(temperature) from cassandra.test allow filtering; +select sum(temperature) from cassandra.test where devgroup<10 allow filtering; +select sum(temperature) from cassandra.test where devgroup<20 allow filtering; +select sum(temperature) from cassandra.test where devgroup<30 allow filtering; +select sum(temperature) from cassandra.test where devgroup<40 allow filtering; +select sum(temperature) from cassandra.test where devgroup<50 allow filtering; +select sum(temperature) from cassandra.test where devgroup<60 allow filtering; +select sum(temperature) from cassandra.test where devgroup<70 allow filtering; +select sum(temperature) from cassandra.test where devgroup<80 allow filtering; +select sum(temperature) from cassandra.test where devgroup<90 allow filtering; +select sum(temperature) from cassandra.test allow filtering; +select max(temperature) from cassandra.test where devgroup<10 allow filtering; +select max(temperature) from cassandra.test where devgroup<20 allow filtering; +select max(temperature) from cassandra.test where devgroup<30 allow filtering; +select max(temperature) from cassandra.test where devgroup<40 allow filtering; +select max(temperature) from cassandra.test where devgroup<50 allow filtering; +select max(temperature) from cassandra.test where devgroup<60 allow filtering; +select max(temperature) from cassandra.test where devgroup<70 allow filtering; +select max(temperature) from cassandra.test where devgroup<80 allow filtering; +select max(temperature) from cassandra.test where devgroup<90 allow filtering; +select max(temperature) from cassandra.test allow filtering; +select min(temperature) from cassandra.test where devgroup<10 allow filtering; +select min(temperature) from cassandra.test where devgroup<20 allow filtering; +select min(temperature) from cassandra.test where devgroup<30 allow filtering; +select min(temperature) from cassandra.test where devgroup<40 allow filtering; +select min(temperature) from cassandra.test where devgroup<50 allow filtering; +select min(temperature) from cassandra.test where devgroup<60 allow filtering; +select min(temperature) from cassandra.test where devgroup<70 allow filtering; +select min(temperature) from cassandra.test where devgroup<80 allow filtering; +select min(temperature) from cassandra.test where devgroup<90 allow filtering; +select min(temperature) from cassandra.test allow filtering; diff --git a/tests/comparisonTest/cassandra/q3.txt b/tests/comparisonTest/cassandra/q3.txt index a65fde589e..f1c4e444f4 100644 --- a/tests/comparisonTest/cassandra/q3.txt +++ b/tests/comparisonTest/cassandra/q3.txt @@ -1,10 +1,10 @@ -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<10 group by devgroup allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<20 group by devgroup allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<30 group by devgroup allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<40 group by devgroup allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<50 group by devgroup allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<60 group by devgroup allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<70 group by devgroup allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<80 group by devgroup allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<90 group by devgroup allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test group by devgroup allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<10 group by devgroup allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<20 group by devgroup allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<30 group by devgroup allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<40 group by devgroup allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<50 group by devgroup allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<60 group by devgroup allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<70 group by devgroup allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<80 group by devgroup allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<90 group by devgroup allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test group by devgroup allow filtering; diff --git a/tests/comparisonTest/cassandra/q4.txt b/tests/comparisonTest/cassandra/q4.txt index 74d4e84f9a..bfba710e05 100644 --- a/tests/comparisonTest/cassandra/q4.txt +++ b/tests/comparisonTest/cassandra/q4.txt @@ -1,10 +1,10 @@ -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<10 group by minute allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<20 group by minute allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<30 group by minute allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<40 group by minute allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<50 group by minute allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<60 group by minute allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<70 group by minute allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<80 group by minute allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<90 group by minute allow filtering; -select count(temperature), sum(temperature), avg(temperature) from cassandra.test group by minute; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<10 group by minute allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<20 group by minute allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<30 group by minute allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<40 group by minute allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<50 group by minute allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<60 group by minute allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<70 group by minute allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<80 group by minute allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test where devgroup<90 group by minute allow filtering; +select count(temperature), sum(temperature), avg(temperature) from cassandra.test group by minute; diff --git a/tests/comparisonTest/influxdb/main.go b/tests/comparisonTest/influxdb/main.go index 2b8fdbeaca..a6550013e3 100644 --- a/tests/comparisonTest/influxdb/main.go +++ b/tests/comparisonTest/influxdb/main.go @@ -1,279 +1,279 @@ -package main - -import ( - "bufio" - "flag" - "fmt" - "log" - "os" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/influxdata/influxdb1-client/v2" -) - -type ProArgs struct { - host string - username string - password string - db string - sql string - dataDir string - filesNum int - writeClients int - rowsPerRequest int -} - -type WriteInfo struct { - threadId int - sID int - eID int -} - -type StatisInfo struct { - totalRows int64 -} - -var statis StatisInfo - -func main() { - // Configuration - var arguments ProArgs - - // Parse options - flag.StringVar(&(arguments.host), "host", "http://localhost:8086", "Server host to connect") - flag.StringVar(&(arguments.db), "db", "db", "DB to insert data") - flag.StringVar(&(arguments.username), "user", "", "Username used to connect to server") - flag.StringVar(&(arguments.password), "pass", "", "Password used to connect to server") - flag.StringVar(&(arguments.sql), "sql", "./sqlCmd.txt", "File name of SQL commands") - flag.StringVar(&(arguments.dataDir), "dataDir", "./testdata", "Raw csv data") - flag.IntVar(&(arguments.filesNum), "numOfFiles", 10, "Number of files int dataDir ") - flag.IntVar(&(arguments.writeClients), "writeClients", 0, "Number of write clients") - flag.IntVar(&(arguments.rowsPerRequest), "rowsPerRequest", 100, "Number of rows per request") - - flag.Parse() - statis.totalRows = 0 - - if arguments.writeClients > 0 { - writeData(&arguments) - } else { - readData(&arguments) - } -} - -func writeData(arguments *ProArgs) { - log.Println("write data") - log.Println("---- writeClients:", arguments.writeClients) - log.Println("---- dataDir:", arguments.dataDir) - log.Println("---- numOfFiles:", arguments.filesNum) - log.Println("---- rowsPerRequest:", arguments.rowsPerRequest) - - var wg sync.WaitGroup - wg.Add(arguments.writeClients) - - st := time.Now() - - a := arguments.filesNum / arguments.writeClients - b := arguments.filesNum % arguments.writeClients - last := 0 - for i := 0; i < arguments.writeClients; i++ { - var wInfo WriteInfo - wInfo.threadId = i + 1 - wInfo.sID = last - if i < b { - wInfo.eID = last + a - } else { - wInfo.eID = last + a - 1 - } - last = wInfo.eID + 1 - go writeDataImp(&wInfo, &wg, arguments) - } - - wg.Wait() - - elapsed := time.Since(st) - seconds := float64(elapsed) / float64(time.Second) - - log.Println("---- Spent", seconds, "seconds to insert", statis.totalRows, "records, speed:", float64(statis.totalRows)/seconds, "Rows/Second") -} - -func writeDataImp(wInfo *WriteInfo, wg *sync.WaitGroup, arguments *ProArgs) { - defer wg.Done() - - log.Println("Thread", wInfo.threadId, "writing sID", wInfo.sID, "eID", wInfo.eID) - - // Connect to the server - conn, err := client.NewHTTPClient(client.HTTPConfig{ - Addr: arguments.host, - Username: arguments.username, - Password: arguments.password, - }) - - if err != nil { - log.Fatal(err) - } - - defer conn.Close() - - // Create database - _, err = queryDB(conn, fmt.Sprintf("create database %s", arguments.db), arguments.db) - if err != nil { - log.Fatal(err) - } - - // Write data - counter := 0 - totalRecords := 0 - - bp, err := client.NewBatchPoints(client.BatchPointsConfig{ - Database: arguments.db, - Precision: "ms", - }) - if err != nil { - log.Fatal(err) - } - - for j := wInfo.sID; j <= wInfo.eID; j++ { - fileName := fmt.Sprintf("%s/testdata%d.csv", arguments.dataDir, j) - fs, err := os.Open(fileName) - if err != nil { - log.Printf("failed to open file %s", fileName) - log.Fatal(err) - } - log.Printf("open file %s success", fileName) - - bfRd := bufio.NewReader(fs) - for { - sline, err := bfRd.ReadString('\n') - if err != nil { - break - } - - sline = strings.TrimSuffix(sline, "\n") - s := strings.Split(sline, " ") - if len(s) != 6 { - continue - } - - // Create a point and add to batch - tags := map[string]string{ - "devid": s[0], - "devname": s[1], - "devgroup": s[2], - } - - timestamp, _ := strconv.ParseInt(s[3], 10, 64) - temperature, _ := strconv.ParseInt(s[4], 10, 32) - humidity, _ := strconv.ParseFloat(s[5], 64) - - fields := map[string]interface{}{ - "temperature": temperature, - "humidity": humidity, - } - - pt, err := client.NewPoint("devices", tags, fields, time.Unix(0, timestamp * int64(time.Millisecond))) - if err != nil { - log.Fatalln("Error: ", err) - } - - bp.AddPoint(pt) - counter++ - - if counter >= arguments.rowsPerRequest { - if err := conn.Write(bp); err != nil { - log.Fatal(err) - } - - totalRecords += counter - counter = 0 - bp, err = client.NewBatchPoints(client.BatchPointsConfig{ - Database: arguments.db, - Precision: "ms", - }) - if err != nil { - log.Fatal(err) - } - } - } - - fs.Close() - } - - totalRecords += counter - if counter > 0 { - if err := conn.Write(bp); err != nil { - log.Fatal(err) - } - } - - atomic.AddInt64(&statis.totalRows, int64(totalRecords)) -} - -func readData(arguments *ProArgs) { - log.Println("read data") - log.Println("---- sql:", arguments.sql) - - conn, err := client.NewHTTPClient(client.HTTPConfig{ - Addr: arguments.host, - Username: arguments.username, - Password: arguments.password, - }) - - if err != nil { - log.Fatal(err) - } - - defer conn.Close() - - fs, err := os.Open(arguments.sql) - if err != nil { - log.Printf("failed to open file %s", arguments.sql) - log.Fatal(err) - } - log.Printf("open file %s success", arguments.sql) - - bfRd := bufio.NewReader(fs) - - for { - sline, err := bfRd.ReadString('\n') - if err != nil { - break - } - - sline = strings.TrimSuffix(sline, "\n") - - st := time.Now() - - _, err = queryDB(conn, sline, arguments.db) - if err != nil { - log.Fatal(err) - } - - elapsed := time.Since(st) - seconds := float64(elapsed) / float64(time.Second) - - log.Println("---- Spent", seconds, "seconds to query ", sline) - } -} - -func queryDB(conn client.Client, cmd string, db string) (res []client.Result, err error) { - query := client.Query{ - Command: cmd, - Database: db, - } - - response, err := conn.Query(query) - if err == nil { - if response.Error() != nil { - return res, response.Error() - } - res = response.Results - } else { - return res, err - } - - return res, nil -} +package main + +import ( + "bufio" + "flag" + "fmt" + "log" + "os" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/influxdata/influxdb1-client/v2" +) + +type ProArgs struct { + host string + username string + password string + db string + sql string + dataDir string + filesNum int + writeClients int + rowsPerRequest int +} + +type WriteInfo struct { + threadId int + sID int + eID int +} + +type StatisInfo struct { + totalRows int64 +} + +var statis StatisInfo + +func main() { + // Configuration + var arguments ProArgs + + // Parse options + flag.StringVar(&(arguments.host), "host", "http://localhost:8086", "Server host to connect") + flag.StringVar(&(arguments.db), "db", "db", "DB to insert data") + flag.StringVar(&(arguments.username), "user", "", "Username used to connect to server") + flag.StringVar(&(arguments.password), "pass", "", "Password used to connect to server") + flag.StringVar(&(arguments.sql), "sql", "./sqlCmd.txt", "File name of SQL commands") + flag.StringVar(&(arguments.dataDir), "dataDir", "./testdata", "Raw csv data") + flag.IntVar(&(arguments.filesNum), "numOfFiles", 10, "Number of files int dataDir ") + flag.IntVar(&(arguments.writeClients), "writeClients", 0, "Number of write clients") + flag.IntVar(&(arguments.rowsPerRequest), "rowsPerRequest", 100, "Number of rows per request") + + flag.Parse() + statis.totalRows = 0 + + if arguments.writeClients > 0 { + writeData(&arguments) + } else { + readData(&arguments) + } +} + +func writeData(arguments *ProArgs) { + log.Println("write data") + log.Println("---- writeClients:", arguments.writeClients) + log.Println("---- dataDir:", arguments.dataDir) + log.Println("---- numOfFiles:", arguments.filesNum) + log.Println("---- rowsPerRequest:", arguments.rowsPerRequest) + + var wg sync.WaitGroup + wg.Add(arguments.writeClients) + + st := time.Now() + + a := arguments.filesNum / arguments.writeClients + b := arguments.filesNum % arguments.writeClients + last := 0 + for i := 0; i < arguments.writeClients; i++ { + var wInfo WriteInfo + wInfo.threadId = i + 1 + wInfo.sID = last + if i < b { + wInfo.eID = last + a + } else { + wInfo.eID = last + a - 1 + } + last = wInfo.eID + 1 + go writeDataImp(&wInfo, &wg, arguments) + } + + wg.Wait() + + elapsed := time.Since(st) + seconds := float64(elapsed) / float64(time.Second) + + log.Println("---- Spent", seconds, "seconds to insert", statis.totalRows, "records, speed:", float64(statis.totalRows)/seconds, "Rows/Second") +} + +func writeDataImp(wInfo *WriteInfo, wg *sync.WaitGroup, arguments *ProArgs) { + defer wg.Done() + + log.Println("Thread", wInfo.threadId, "writing sID", wInfo.sID, "eID", wInfo.eID) + + // Connect to the server + conn, err := client.NewHTTPClient(client.HTTPConfig{ + Addr: arguments.host, + Username: arguments.username, + Password: arguments.password, + }) + + if err != nil { + log.Fatal(err) + } + + defer conn.Close() + + // Create database + _, err = queryDB(conn, fmt.Sprintf("create database %s", arguments.db), arguments.db) + if err != nil { + log.Fatal(err) + } + + // Write data + counter := 0 + totalRecords := 0 + + bp, err := client.NewBatchPoints(client.BatchPointsConfig{ + Database: arguments.db, + Precision: "ms", + }) + if err != nil { + log.Fatal(err) + } + + for j := wInfo.sID; j <= wInfo.eID; j++ { + fileName := fmt.Sprintf("%s/testdata%d.csv", arguments.dataDir, j) + fs, err := os.Open(fileName) + if err != nil { + log.Printf("failed to open file %s", fileName) + log.Fatal(err) + } + log.Printf("open file %s success", fileName) + + bfRd := bufio.NewReader(fs) + for { + sline, err := bfRd.ReadString('\n') + if err != nil { + break + } + + sline = strings.TrimSuffix(sline, "\n") + s := strings.Split(sline, " ") + if len(s) != 6 { + continue + } + + // Create a point and add to batch + tags := map[string]string{ + "devid": s[0], + "devname": s[1], + "devgroup": s[2], + } + + timestamp, _ := strconv.ParseInt(s[3], 10, 64) + temperature, _ := strconv.ParseInt(s[4], 10, 32) + humidity, _ := strconv.ParseFloat(s[5], 64) + + fields := map[string]interface{}{ + "temperature": temperature, + "humidity": humidity, + } + + pt, err := client.NewPoint("devices", tags, fields, time.Unix(0, timestamp * int64(time.Millisecond))) + if err != nil { + log.Fatalln("Error: ", err) + } + + bp.AddPoint(pt) + counter++ + + if counter >= arguments.rowsPerRequest { + if err := conn.Write(bp); err != nil { + log.Fatal(err) + } + + totalRecords += counter + counter = 0 + bp, err = client.NewBatchPoints(client.BatchPointsConfig{ + Database: arguments.db, + Precision: "ms", + }) + if err != nil { + log.Fatal(err) + } + } + } + + fs.Close() + } + + totalRecords += counter + if counter > 0 { + if err := conn.Write(bp); err != nil { + log.Fatal(err) + } + } + + atomic.AddInt64(&statis.totalRows, int64(totalRecords)) +} + +func readData(arguments *ProArgs) { + log.Println("read data") + log.Println("---- sql:", arguments.sql) + + conn, err := client.NewHTTPClient(client.HTTPConfig{ + Addr: arguments.host, + Username: arguments.username, + Password: arguments.password, + }) + + if err != nil { + log.Fatal(err) + } + + defer conn.Close() + + fs, err := os.Open(arguments.sql) + if err != nil { + log.Printf("failed to open file %s", arguments.sql) + log.Fatal(err) + } + log.Printf("open file %s success", arguments.sql) + + bfRd := bufio.NewReader(fs) + + for { + sline, err := bfRd.ReadString('\n') + if err != nil { + break + } + + sline = strings.TrimSuffix(sline, "\n") + + st := time.Now() + + _, err = queryDB(conn, sline, arguments.db) + if err != nil { + log.Fatal(err) + } + + elapsed := time.Since(st) + seconds := float64(elapsed) / float64(time.Second) + + log.Println("---- Spent", seconds, "seconds to query ", sline) + } +} + +func queryDB(conn client.Client, cmd string, db string) (res []client.Result, err error) { + query := client.Query{ + Command: cmd, + Database: db, + } + + response, err := conn.Query(query) + if err == nil { + if response.Error() != nil { + return res, response.Error() + } + res = response.Results + } else { + return res, err + } + + return res, nil +} diff --git a/tests/comparisonTest/influxdb/q1.txt b/tests/comparisonTest/influxdb/q1.txt index 4a485508cf..4bff11d200 100644 --- a/tests/comparisonTest/influxdb/q1.txt +++ b/tests/comparisonTest/influxdb/q1.txt @@ -1,11 +1,11 @@ -select * from devices where devgroup='0'; -select * from devices where devgroup='10'; -select * from devices where devgroup='20'; -select * from devices where devgroup='30'; -select * from devices where devgroup='40'; -select * from devices where devgroup='50'; -select * from devices where devgroup='60'; -select * from devices where devgroup='70'; -select * from devices where devgroup='80'; -select * from devices where devgroup='90'; - +select * from devices where devgroup='0'; +select * from devices where devgroup='10'; +select * from devices where devgroup='20'; +select * from devices where devgroup='30'; +select * from devices where devgroup='40'; +select * from devices where devgroup='50'; +select * from devices where devgroup='60'; +select * from devices where devgroup='70'; +select * from devices where devgroup='80'; +select * from devices where devgroup='90'; + diff --git a/tests/comparisonTest/influxdb/q2.txt b/tests/comparisonTest/influxdb/q2.txt index 4b02dd6b94..c271837ede 100644 --- a/tests/comparisonTest/influxdb/q2.txt +++ b/tests/comparisonTest/influxdb/q2.txt @@ -1,61 +1,61 @@ -select count(temperature) from devices where devgroup=~/[1-1][0-9]/; -select count(temperature) from devices where devgroup=~/[1-2][0-9]/; -select count(temperature) from devices where devgroup=~/[1-3][0-9]/; -select count(temperature) from devices where devgroup=~/[1-4][0-9]/; -select count(temperature) from devices where devgroup=~/[1-5][0-9]/; -select count(temperature) from devices where devgroup=~/[1-6][0-9]/; -select count(temperature) from devices where devgroup=~/[1-7][0-9]/; -select count(temperature) from devices where devgroup=~/[1-8][0-9]/; -select count(temperature) from devices where devgroup=~/[1-9][0-9]/; -select count(temperature) from devices; -select mean(temperature) from devices where devgroup=~/[1-1][0-9]/; -select mean(temperature) from devices where devgroup=~/[1-2][0-9]/; -select mean(temperature) from devices where devgroup=~/[1-3][0-9]/; -select mean(temperature) from devices where devgroup=~/[1-4][0-9]/; -select mean(temperature) from devices where devgroup=~/[1-5][0-9]/; -select mean(temperature) from devices where devgroup=~/[1-6][0-9]/; -select mean(temperature) from devices where devgroup=~/[1-7][0-9]/; -select mean(temperature) from devices where devgroup=~/[1-8][0-9]/; -select mean(temperature) from devices where devgroup=~/[1-9][0-9]/; -select mean(temperature) from devices; -select sum(temperature) from devices where devgroup=~/[1-1][0-9]/; -select sum(temperature) from devices where devgroup=~/[1-2][0-9]/; -select sum(temperature) from devices where devgroup=~/[1-3][0-9]/; -select sum(temperature) from devices where devgroup=~/[1-4][0-9]/; -select sum(temperature) from devices where devgroup=~/[1-5][0-9]/; -select sum(temperature) from devices where devgroup=~/[1-6][0-9]/; -select sum(temperature) from devices where devgroup=~/[1-7][0-9]/; -select sum(temperature) from devices where devgroup=~/[1-8][0-9]/; -select sum(temperature) from devices where devgroup=~/[1-9][0-9]/; -select sum(temperature) from devices; -select max(temperature) from devices where devgroup=~/[1-1][0-9]/; -select max(temperature) from devices where devgroup=~/[1-2][0-9]/; -select max(temperature) from devices where devgroup=~/[1-3][0-9]/; -select max(temperature) from devices where devgroup=~/[1-4][0-9]/; -select max(temperature) from devices where devgroup=~/[1-5][0-9]/; -select max(temperature) from devices where devgroup=~/[1-6][0-9]/; -select max(temperature) from devices where devgroup=~/[1-7][0-9]/; -select max(temperature) from devices where devgroup=~/[1-8][0-9]/; -select max(temperature) from devices where devgroup=~/[1-9][0-9]/; -select max(temperature) from devices; -select min(temperature) from devices where devgroup=~/[1-1][0-9]/; -select min(temperature) from devices where devgroup=~/[1-2][0-9]/; -select min(temperature) from devices where devgroup=~/[1-3][0-9]/; -select min(temperature) from devices where devgroup=~/[1-4][0-9]/; -select min(temperature) from devices where devgroup=~/[1-5][0-9]/; -select min(temperature) from devices where devgroup=~/[1-6][0-9]/; -select min(temperature) from devices where devgroup=~/[1-7][0-9]/; -select min(temperature) from devices where devgroup=~/[1-8][0-9]/; -select min(temperature) from devices where devgroup=~/[1-9][0-9]/; -select min(temperature) from devices; -select spread(temperature) from devices where devgroup=~/[1-1][0-9]/; -select spread(temperature) from devices where devgroup=~/[1-2][0-9]/; -select spread(temperature) from devices where devgroup=~/[1-3][0-9]/; -select spread(temperature) from devices where devgroup=~/[1-4][0-9]/; -select spread(temperature) from devices where devgroup=~/[1-5][0-9]/; -select spread(temperature) from devices where devgroup=~/[1-6][0-9]/; -select spread(temperature) from devices where devgroup=~/[1-7][0-9]/; -select spread(temperature) from devices where devgroup=~/[1-8][0-9]/; -select spread(temperature) from devices where devgroup=~/[1-9][0-9]/; -select spread(temperature) from devices; - +select count(temperature) from devices where devgroup=~/[1-1][0-9]/; +select count(temperature) from devices where devgroup=~/[1-2][0-9]/; +select count(temperature) from devices where devgroup=~/[1-3][0-9]/; +select count(temperature) from devices where devgroup=~/[1-4][0-9]/; +select count(temperature) from devices where devgroup=~/[1-5][0-9]/; +select count(temperature) from devices where devgroup=~/[1-6][0-9]/; +select count(temperature) from devices where devgroup=~/[1-7][0-9]/; +select count(temperature) from devices where devgroup=~/[1-8][0-9]/; +select count(temperature) from devices where devgroup=~/[1-9][0-9]/; +select count(temperature) from devices; +select mean(temperature) from devices where devgroup=~/[1-1][0-9]/; +select mean(temperature) from devices where devgroup=~/[1-2][0-9]/; +select mean(temperature) from devices where devgroup=~/[1-3][0-9]/; +select mean(temperature) from devices where devgroup=~/[1-4][0-9]/; +select mean(temperature) from devices where devgroup=~/[1-5][0-9]/; +select mean(temperature) from devices where devgroup=~/[1-6][0-9]/; +select mean(temperature) from devices where devgroup=~/[1-7][0-9]/; +select mean(temperature) from devices where devgroup=~/[1-8][0-9]/; +select mean(temperature) from devices where devgroup=~/[1-9][0-9]/; +select mean(temperature) from devices; +select sum(temperature) from devices where devgroup=~/[1-1][0-9]/; +select sum(temperature) from devices where devgroup=~/[1-2][0-9]/; +select sum(temperature) from devices where devgroup=~/[1-3][0-9]/; +select sum(temperature) from devices where devgroup=~/[1-4][0-9]/; +select sum(temperature) from devices where devgroup=~/[1-5][0-9]/; +select sum(temperature) from devices where devgroup=~/[1-6][0-9]/; +select sum(temperature) from devices where devgroup=~/[1-7][0-9]/; +select sum(temperature) from devices where devgroup=~/[1-8][0-9]/; +select sum(temperature) from devices where devgroup=~/[1-9][0-9]/; +select sum(temperature) from devices; +select max(temperature) from devices where devgroup=~/[1-1][0-9]/; +select max(temperature) from devices where devgroup=~/[1-2][0-9]/; +select max(temperature) from devices where devgroup=~/[1-3][0-9]/; +select max(temperature) from devices where devgroup=~/[1-4][0-9]/; +select max(temperature) from devices where devgroup=~/[1-5][0-9]/; +select max(temperature) from devices where devgroup=~/[1-6][0-9]/; +select max(temperature) from devices where devgroup=~/[1-7][0-9]/; +select max(temperature) from devices where devgroup=~/[1-8][0-9]/; +select max(temperature) from devices where devgroup=~/[1-9][0-9]/; +select max(temperature) from devices; +select min(temperature) from devices where devgroup=~/[1-1][0-9]/; +select min(temperature) from devices where devgroup=~/[1-2][0-9]/; +select min(temperature) from devices where devgroup=~/[1-3][0-9]/; +select min(temperature) from devices where devgroup=~/[1-4][0-9]/; +select min(temperature) from devices where devgroup=~/[1-5][0-9]/; +select min(temperature) from devices where devgroup=~/[1-6][0-9]/; +select min(temperature) from devices where devgroup=~/[1-7][0-9]/; +select min(temperature) from devices where devgroup=~/[1-8][0-9]/; +select min(temperature) from devices where devgroup=~/[1-9][0-9]/; +select min(temperature) from devices; +select spread(temperature) from devices where devgroup=~/[1-1][0-9]/; +select spread(temperature) from devices where devgroup=~/[1-2][0-9]/; +select spread(temperature) from devices where devgroup=~/[1-3][0-9]/; +select spread(temperature) from devices where devgroup=~/[1-4][0-9]/; +select spread(temperature) from devices where devgroup=~/[1-5][0-9]/; +select spread(temperature) from devices where devgroup=~/[1-6][0-9]/; +select spread(temperature) from devices where devgroup=~/[1-7][0-9]/; +select spread(temperature) from devices where devgroup=~/[1-8][0-9]/; +select spread(temperature) from devices where devgroup=~/[1-9][0-9]/; +select spread(temperature) from devices; + diff --git a/tests/comparisonTest/influxdb/q3.txt b/tests/comparisonTest/influxdb/q3.txt index 75a42c2553..fd32f3ea66 100644 --- a/tests/comparisonTest/influxdb/q3.txt +++ b/tests/comparisonTest/influxdb/q3.txt @@ -1,11 +1,11 @@ -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-1][0-9]/ group by devgroup; -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-2][0-9]/ group by devgroup; -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-3][0-9]/ group by devgroup; -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-4][0-9]/ group by devgroup; -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-5][0-9]/ group by devgroup; -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-6][0-9]/ group by devgroup; -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-7][0-9]/ group by devgroup; -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-8][0-9]/ group by devgroup; -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-9][0-9]/ group by devgroup; -select count(temperature), sum(temperature), mean(temperature) from devices group by devgroup; - +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-1][0-9]/ group by devgroup; +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-2][0-9]/ group by devgroup; +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-3][0-9]/ group by devgroup; +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-4][0-9]/ group by devgroup; +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-5][0-9]/ group by devgroup; +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-6][0-9]/ group by devgroup; +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-7][0-9]/ group by devgroup; +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-8][0-9]/ group by devgroup; +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-9][0-9]/ group by devgroup; +select count(temperature), sum(temperature), mean(temperature) from devices group by devgroup; + diff --git a/tests/comparisonTest/influxdb/q4.txt b/tests/comparisonTest/influxdb/q4.txt index 90ce7da2d6..5b1524d607 100644 --- a/tests/comparisonTest/influxdb/q4.txt +++ b/tests/comparisonTest/influxdb/q4.txt @@ -1,11 +1,11 @@ -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-1][0-9]/ group by time(1m); -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-2][0-9]/ group by time(1m); -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-3][0-9]/ group by time(1m); -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-4][0-9]/ group by time(1m); -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-5][0-9]/ group by time(1m); -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-6][0-9]/ group by time(1m); -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-7][0-9]/ group by time(1m); -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-8][0-9]/ group by time(1m); -select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-9][0-9]/ group by time(1m); -select count(temperature), sum(temperature), mean(temperature) from devices group by time(1m); - +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-1][0-9]/ group by time(1m); +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-2][0-9]/ group by time(1m); +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-3][0-9]/ group by time(1m); +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-4][0-9]/ group by time(1m); +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-5][0-9]/ group by time(1m); +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-6][0-9]/ group by time(1m); +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-7][0-9]/ group by time(1m); +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-8][0-9]/ group by time(1m); +select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-9][0-9]/ group by time(1m); +select count(temperature), sum(temperature), mean(temperature) from devices group by time(1m); + diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml index 6f5fa6e132..9271d478a6 100644 --- a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml +++ b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml @@ -118,7 +118,7 @@ com.fasterxml.jackson.core jackson-databind - 2.9.10.1 + 2.9.10 diff --git a/tests/comparisonTest/tdengine/q1.txt b/tests/comparisonTest/tdengine/q1.txt index f2a4010fdf..dbe5d78a3f 100644 --- a/tests/comparisonTest/tdengine/q1.txt +++ b/tests/comparisonTest/tdengine/q1.txt @@ -1,11 +1,11 @@ -select * from db.devices where devgroup=0; -select * from db.devices where devgroup=10; -select * from db.devices where devgroup=20; -select * from db.devices where devgroup=30; -select * from db.devices where devgroup=40; -select * from db.devices where devgroup=50; -select * from db.devices where devgroup=60; -select * from db.devices where devgroup=70; -select * from db.devices where devgroup=80; -select * from db.devices where devgroup=90; - +select * from db.devices where devgroup=0; +select * from db.devices where devgroup=10; +select * from db.devices where devgroup=20; +select * from db.devices where devgroup=30; +select * from db.devices where devgroup=40; +select * from db.devices where devgroup=50; +select * from db.devices where devgroup=60; +select * from db.devices where devgroup=70; +select * from db.devices where devgroup=80; +select * from db.devices where devgroup=90; + diff --git a/tests/comparisonTest/tdengine/q2.txt b/tests/comparisonTest/tdengine/q2.txt index 0ed7686e93..95d8a9e261 100644 --- a/tests/comparisonTest/tdengine/q2.txt +++ b/tests/comparisonTest/tdengine/q2.txt @@ -1,61 +1,61 @@ -select count(*) from db.devices where devgroup<10; -select count(*) from db.devices where devgroup<20; -select count(*) from db.devices where devgroup<30; -select count(*) from db.devices where devgroup<40; -select count(*) from db.devices where devgroup<50; -select count(*) from db.devices where devgroup<60; -select count(*) from db.devices where devgroup<70; -select count(*) from db.devices where devgroup<80; -select count(*) from db.devices where devgroup<90; -select count(*) from db.devices; -select avg(temperature) from db.devices where devgroup<10; -select avg(temperature) from db.devices where devgroup<20; -select avg(temperature) from db.devices where devgroup<30; -select avg(temperature) from db.devices where devgroup<40; -select avg(temperature) from db.devices where devgroup<50; -select avg(temperature) from db.devices where devgroup<60; -select avg(temperature) from db.devices where devgroup<70; -select avg(temperature) from db.devices where devgroup<80; -select avg(temperature) from db.devices where devgroup<90; -select avg(temperature) from db.devices; -select sum(temperature) from db.devices where devgroup<10; -select sum(temperature) from db.devices where devgroup<20; -select sum(temperature) from db.devices where devgroup<30; -select sum(temperature) from db.devices where devgroup<40; -select sum(temperature) from db.devices where devgroup<50; -select sum(temperature) from db.devices where devgroup<60; -select sum(temperature) from db.devices where devgroup<70; -select sum(temperature) from db.devices where devgroup<80; -select sum(temperature) from db.devices where devgroup<90; -select sum(temperature) from db.devices; -select max(temperature) from db.devices where devgroup<10; -select max(temperature) from db.devices where devgroup<20; -select max(temperature) from db.devices where devgroup<30; -select max(temperature) from db.devices where devgroup<40; -select max(temperature) from db.devices where devgroup<50; -select max(temperature) from db.devices where devgroup<60; -select max(temperature) from db.devices where devgroup<70; -select max(temperature) from db.devices where devgroup<80; -select max(temperature) from db.devices where devgroup<90; -select max(temperature) from db.devices; -select min(temperature) from db.devices where devgroup<10; -select min(temperature) from db.devices where devgroup<20; -select min(temperature) from db.devices where devgroup<30; -select min(temperature) from db.devices where devgroup<40; -select min(temperature) from db.devices where devgroup<50; -select min(temperature) from db.devices where devgroup<60; -select min(temperature) from db.devices where devgroup<70; -select min(temperature) from db.devices where devgroup<80; -select min(temperature) from db.devices where devgroup<90; -select min(temperature) from db.devices; -select spread(temperature) from db.devices where devgroup<10; -select spread(temperature) from db.devices where devgroup<20; -select spread(temperature) from db.devices where devgroup<30; -select spread(temperature) from db.devices where devgroup<40; -select spread(temperature) from db.devices where devgroup<50; -select spread(temperature) from db.devices where devgroup<60; -select spread(temperature) from db.devices where devgroup<70; -select spread(temperature) from db.devices where devgroup<80; -select spread(temperature) from db.devices where devgroup<90; -select spread(temperature) from db.devices; - +select count(*) from db.devices where devgroup<10; +select count(*) from db.devices where devgroup<20; +select count(*) from db.devices where devgroup<30; +select count(*) from db.devices where devgroup<40; +select count(*) from db.devices where devgroup<50; +select count(*) from db.devices where devgroup<60; +select count(*) from db.devices where devgroup<70; +select count(*) from db.devices where devgroup<80; +select count(*) from db.devices where devgroup<90; +select count(*) from db.devices; +select avg(temperature) from db.devices where devgroup<10; +select avg(temperature) from db.devices where devgroup<20; +select avg(temperature) from db.devices where devgroup<30; +select avg(temperature) from db.devices where devgroup<40; +select avg(temperature) from db.devices where devgroup<50; +select avg(temperature) from db.devices where devgroup<60; +select avg(temperature) from db.devices where devgroup<70; +select avg(temperature) from db.devices where devgroup<80; +select avg(temperature) from db.devices where devgroup<90; +select avg(temperature) from db.devices; +select sum(temperature) from db.devices where devgroup<10; +select sum(temperature) from db.devices where devgroup<20; +select sum(temperature) from db.devices where devgroup<30; +select sum(temperature) from db.devices where devgroup<40; +select sum(temperature) from db.devices where devgroup<50; +select sum(temperature) from db.devices where devgroup<60; +select sum(temperature) from db.devices where devgroup<70; +select sum(temperature) from db.devices where devgroup<80; +select sum(temperature) from db.devices where devgroup<90; +select sum(temperature) from db.devices; +select max(temperature) from db.devices where devgroup<10; +select max(temperature) from db.devices where devgroup<20; +select max(temperature) from db.devices where devgroup<30; +select max(temperature) from db.devices where devgroup<40; +select max(temperature) from db.devices where devgroup<50; +select max(temperature) from db.devices where devgroup<60; +select max(temperature) from db.devices where devgroup<70; +select max(temperature) from db.devices where devgroup<80; +select max(temperature) from db.devices where devgroup<90; +select max(temperature) from db.devices; +select min(temperature) from db.devices where devgroup<10; +select min(temperature) from db.devices where devgroup<20; +select min(temperature) from db.devices where devgroup<30; +select min(temperature) from db.devices where devgroup<40; +select min(temperature) from db.devices where devgroup<50; +select min(temperature) from db.devices where devgroup<60; +select min(temperature) from db.devices where devgroup<70; +select min(temperature) from db.devices where devgroup<80; +select min(temperature) from db.devices where devgroup<90; +select min(temperature) from db.devices; +select spread(temperature) from db.devices where devgroup<10; +select spread(temperature) from db.devices where devgroup<20; +select spread(temperature) from db.devices where devgroup<30; +select spread(temperature) from db.devices where devgroup<40; +select spread(temperature) from db.devices where devgroup<50; +select spread(temperature) from db.devices where devgroup<60; +select spread(temperature) from db.devices where devgroup<70; +select spread(temperature) from db.devices where devgroup<80; +select spread(temperature) from db.devices where devgroup<90; +select spread(temperature) from db.devices; + diff --git a/tests/comparisonTest/tdengine/q3.txt b/tests/comparisonTest/tdengine/q3.txt index b042640e62..e2def2d66c 100644 --- a/tests/comparisonTest/tdengine/q3.txt +++ b/tests/comparisonTest/tdengine/q3.txt @@ -1,11 +1,11 @@ -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<10 group by devgroup; -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<20 group by devgroup; -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<30 group by devgroup; -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<40 group by devgroup; -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<50 group by devgroup; -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<60 group by devgroup; -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<70 group by devgroup; -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<80 group by devgroup; -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<90 group by devgroup; -select count(temperature), sum(temperature), avg(temperature) from db.devices group by devgroup; - +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<10 group by devgroup; +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<20 group by devgroup; +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<30 group by devgroup; +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<40 group by devgroup; +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<50 group by devgroup; +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<60 group by devgroup; +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<70 group by devgroup; +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<80 group by devgroup; +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<90 group by devgroup; +select count(temperature), sum(temperature), avg(temperature) from db.devices group by devgroup; + diff --git a/tests/comparisonTest/tdengine/q4.txt b/tests/comparisonTest/tdengine/q4.txt index 4994dba86c..7fb3a3ffdb 100644 --- a/tests/comparisonTest/tdengine/q4.txt +++ b/tests/comparisonTest/tdengine/q4.txt @@ -1,11 +1,11 @@ -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<10 interval(1m); -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<20 interval(1m); -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<30 interval(1m); -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<40 interval(1m); -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<50 interval(1m); -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<60 interval(1m); -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<70 interval(1m); -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<80 interval(1m); -select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<90 interval(1m); -select count(temperature), sum(temperature), avg(temperature) from db.devices interval(1m); - +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<10 interval(1m); +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<20 interval(1m); +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<30 interval(1m); +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<40 interval(1m); +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<50 interval(1m); +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<60 interval(1m); +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<70 interval(1m); +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<80 interval(1m); +select count(temperature), sum(temperature), avg(temperature) from db.devices where devgroup<90 interval(1m); +select count(temperature), sum(temperature), avg(temperature) from db.devices interval(1m); + diff --git a/tests/examples/R/command.txt b/tests/examples/R/command.txt index 238e9ae880..9a549ff200 100644 --- a/tests/examples/R/command.txt +++ b/tests/examples/R/command.txt @@ -1,55 +1,55 @@ -# Linux Platform -install.packages('rJDBC', repos='http://cran.us.r-project.org') - -# Loading RJDBC packages -library('RJDBC') -# Set up working path and JDBC driver storage location -setwd('C:/TDengine') - -# Load JDBC Driver for TDengine -drv<-JDBC("com.taosdata.jdbc.TSDBDriver","JDBCDriver-1.0.0-dist.jar", identifier.quote="\"") - -# Connect to the database -conn<-dbConnect(drv,"jdbc:TSDB://192.168.1.114:0/?user=root&password=taosdata","root","taosdata") - -# Get connection information -dbGetInfo(conn) - -# Using database test -dbSendUpdate(conn, "use test") - -# Insert data -dbSendUpdate(conn, "insert into t1 values(now, 99)") - -# View all tables -table1<-dbGetQuery(conn,"show tables") - -# Functional support for RJDBC - -# List all tables -dbListTables(conn) - -# Is there table iris -dbExistsTable(conn,”iris”) - -# Connect summary information -summary(conn) -dbGetInfo(conn) - -# Read all the data from the T1 table -dbReadTable(conn, "t1") - -# Delete table t1 -dbRemoveTable(conn,"t1") - -# Execute any non-query SQL statements -dbSendUpdate(conn, "create table t1(a timestamp, b int, c nchar(12))"); - -# Write data -dbWriteTable(conn, "t1", t_demo_n, overwrite=FALSE, append=TRUE) - -# Extracting data on demand using SQL statements -dbGetQuery(conn, "select k from tu") - -# Close the connection -dbDisconnect(conn) +# Linux Platform +install.packages('rJDBC', repos='http://cran.us.r-project.org') + +# Loading RJDBC packages +library('RJDBC') +# Set up working path and JDBC driver storage location +setwd('C:/TDengine') + +# Load JDBC Driver for TDengine +drv<-JDBC("com.taosdata.jdbc.TSDBDriver","JDBCDriver-1.0.0-dist.jar", identifier.quote="\"") + +# Connect to the database +conn<-dbConnect(drv,"jdbc:TSDB://192.168.1.114:0/?user=root&password=taosdata","root","taosdata") + +# Get connection information +dbGetInfo(conn) + +# Using database test +dbSendUpdate(conn, "use test") + +# Insert data +dbSendUpdate(conn, "insert into t1 values(now, 99)") + +# View all tables +table1<-dbGetQuery(conn,"show tables") + +# Functional support for RJDBC + +# List all tables +dbListTables(conn) + +# Is there table iris +dbExistsTable(conn,”iris”) + +# Connect summary information +summary(conn) +dbGetInfo(conn) + +# Read all the data from the T1 table +dbReadTable(conn, "t1") + +# Delete table t1 +dbRemoveTable(conn,"t1") + +# Execute any non-query SQL statements +dbSendUpdate(conn, "create table t1(a timestamp, b int, c nchar(12))"); + +# Write data +dbWriteTable(conn, "t1", t_demo_n, overwrite=FALSE, append=TRUE) + +# Extracting data on demand using SQL statements +dbGetQuery(conn, "select k from tu") + +# Close the connection +dbDisconnect(conn) diff --git a/tests/examples/go/src/taosapp/taosapp.go b/tests/examples/go/src/taosapp/taosapp.go old mode 100755 new mode 100644 diff --git a/tests/examples/lua/build.sh b/tests/examples/lua/build.sh old mode 100755 new mode 100644 diff --git a/tests/examples/matlab/TDengineDemo.m b/tests/examples/matlab/TDengineDemo.m index c02d45e8b9..b44777512b 100644 --- a/tests/examples/matlab/TDengineDemo.m +++ b/tests/examples/matlab/TDengineDemo.m @@ -1,128 +1,128 @@ -%% Connect to TDengine -clear; -fprintf("Connecting to TDengine..."); -dbName = 'tsdb'; -user = 'root'; -password = 'taosdata'; -jdbcDriverName = 'com.taosdata.jdbc.TSDBDriver'; -jdbcUrl = 'jdbc:TSDB://192.168.1.113:0/'; -conn = database(dbName, user, password, jdbcDriverName, jdbcUrl) -if isempty(conn.Message) - fprintf("Connection is successfully established!\n"); -else - fprintf("Failed to connect to server: %s\n", conn.Message); -end - -%% Query a table in TDengine, and store the results in a MATLAB table object 'tb1' -% Please note that the select() function retrieves all rows in a table/supertale into MATLAB -sql = "select ts, distance1 from device1 limit 5"; -fprintf("Execute query: %s\n", sql); -tic -tb1 = select(conn, sql); -timeused = toc; -fprintf("\tQuery completed!\n\tNumber of rows retrieved: %d\n\tNumber of columns in each row: %d\n\tTime used: %g\n", height(tb1), width(tb1), timeused); - -% To go a bit further, we can convert the MATLAB table object to a MATLAB matrix object -data = table2array(tb1) - -%% Query table names in a TDengine supertable, and store the results in a MATLAB table object 'stbmeta' -sql = "select tbname from devices limit 10"; -fprintf("Execute query: %s\n", sql); -tic; -stbmeta = select(conn, sql); -timeused = toc; -fprintf("\tTables in supertable 'devices': %t", stbmeta); -fprintf("\tQuery completed!\n\tNumber of rows retrieved: %d\n\tNumber of columns in each row: %d\n\tTime used: %g\n", height(stbmeta), width(stbmeta), timeused); - -%% Query a TDengine supertable, and stores the results in a MATLAB table object 'stb' -sql = "select ts, distance1 from devices"; -fprintf("Execute query: %s\n", sql); -tic; -stb = select(conn, sql); -timeused = toc; -fprintf("\tQuery completed!\n\tNumber of rows retrieved: %d\n\tNumber of columns in each row: %d\n\tTime used: %g\n", height(stb), width(stb), timeused); - -%% Query TDengine using cursors and specify the number of rows to fetch -sql = 'select * from device1'; -rowLimit = 5; -fprintf("Execute query: %s with row limit set to %d\n", sql, rowLimit); -tic; -% Get cursor -cur = exec(conn, sql); -% Fetch data -cur = fetch(cur, rowLimit); -data = cur.Data -timeused = toc; -fprintf("\tQuery completed!\n\tNumber of rows retrieved: %d\n\tNumber of columns in each row: %d\n\tTime used: %g\n", size(data, 1), size(data, 2), timeused); - -%% Query specific columns in a TDenigine table 'device1', and stores the results directly in a MATLAB cell array 'data' -sql = 'SELECT * FROM device1 order by ts asc'; -fprintf("Execute query: %s\n", sql); -tic; -data = fetch(conn, sql); -timeused = toc; -fprintf("\tQuery completed!\n\tNumber of rows retrieved: %d\n\tNumber of columns in each row: %d\n\tTime used: %g\n", size(data, 1), size(data, 2), timeused); -% Let's now convert the cell array 'data' into some matrices, and make a plot of column 'c1' again the timestamp 'ts' -ts = cell2mat(data(:,1)); -c1 = cell2mat(data(:,2)); - -%% Query aggregation results from a table -% TDengine is so powerful at aggregated computations. Let's calculate the max, mean, standard deviation and min values for every 10 minutes in the -% tb1's timeline, and then plot them together with all the data points in tb1 -sql = sprintf('SELECT max(measure1), avg(measure1), stddev(measure1), min(measure1) FROM device1 WHERE ts >= %d and ts <= %d interval(10m)', ts(1), ts(end)); -fprintf("Execute query: %s\n", sql); -tic; -c1_stats = fetch(conn, sql); -timeused = toc; -fprintf("\tQuery completed!\n\tNumber of rows retrieved: %d\n\tNumber of columns in each row: %d\n\tTime used: %g\n", size(c1_stats, 1), size(c1_stats, 2), timeused); -% Prepare data for plotting. -tsAsDate = datestr(ts/86400000 + datenum(1970,1,1), 'mm-dd HH:MM'); -c1_stats = cell2mat(c1_stats); -c1_stats_ts = c1_stats(:, 1); -c1_stats_max = c1_stats(:, 2); -c1_stats_mean = c1_stats(:, 3); -c1_stats_stddev = c1_stats(:, 4); -c1_stats_min = c1_stats(:, 5); -c1_stats_tsAsDate = datestr(c1_stats(:,1)/86400000 + datenum(1970,1,1), 'mm-dd HH:MM'); - -%% Now let's plot the data and associated statistical aggregation calculation in a figure. -fh = figure(1); -set(fh,'position',[50 50 1300 700]); -h1 = scatter(ts, c1, 5, 'c'); -hold on; -h2 = plot(c1_stats_ts + 300000, c1_stats_max, '-or', 'linewidth', 1); -hold on; -h3 = plot(c1_stats_ts + 300000, c1_stats_mean, '-xg', 'linewidth', 1); -hold on; -h4 = plot(c1_stats_ts + 300000, c1_stats_stddev, '-*y', 'linewidth', 1); -hold on; -h5 = plot(c1_stats_ts + 300000, c1_stats_min, '-+k', 'linewidth', 1); -xlabel('time'); -ylabel('measurement1'); -set(gca, 'xtick',[ts(1),ts(end/4),ts(2*end/4),ts(3*end/4),ts(end)]); -set(gca, 'xticklabel',{tsAsDate(1,:), tsAsDate(end/4,:),tsAsDate(2*end/4,:),tsAsDate(3*end/4,:),tsAsDate(end,:)}); -xlim([ts(1), ts(end)]); -legend([h1, h2, h3, h4, h5], 'data points', 'max per 10 mins', 'mean per 10 mins', 'stddev per 10 mins', 'min per 10 mins'); -title('Device Measurement Monitoring Demo'); -grid on; - -%% Insert data into TDengine using exec() -sql = 'insert into device1 (ts, distance1) values (now, -1)'; -fprintf("Execute query: %s\n", sql); -cur = exec(conn, sql) -sql = 'select * from device1 limit 1'; -fprintf("Execute query: %s\n", sql); -data = select(conn, sql) -conn.close; - -%% Insert data into TDengine using datainsert() -% this is currently not supported - -% colnames = {'ts','c1','c2','c3'}; -% dat = {'now' 99 99 99}; -% tbname = 'plane1'; -% datainsert(conn, tbname, colnames, dat); -% cur = exec(conn, 'select * from ' + tbname); -% cur = fetch(cur, 5); -% data = cur.Data - +%% Connect to TDengine +clear; +fprintf("Connecting to TDengine..."); +dbName = 'tsdb'; +user = 'root'; +password = 'taosdata'; +jdbcDriverName = 'com.taosdata.jdbc.TSDBDriver'; +jdbcUrl = 'jdbc:TSDB://192.168.1.113:0/'; +conn = database(dbName, user, password, jdbcDriverName, jdbcUrl) +if isempty(conn.Message) + fprintf("Connection is successfully established!\n"); +else + fprintf("Failed to connect to server: %s\n", conn.Message); +end + +%% Query a table in TDengine, and store the results in a MATLAB table object 'tb1' +% Please note that the select() function retrieves all rows in a table/supertale into MATLAB +sql = "select ts, distance1 from device1 limit 5"; +fprintf("Execute query: %s\n", sql); +tic +tb1 = select(conn, sql); +timeused = toc; +fprintf("\tQuery completed!\n\tNumber of rows retrieved: %d\n\tNumber of columns in each row: %d\n\tTime used: %g\n", height(tb1), width(tb1), timeused); + +% To go a bit further, we can convert the MATLAB table object to a MATLAB matrix object +data = table2array(tb1) + +%% Query table names in a TDengine supertable, and store the results in a MATLAB table object 'stbmeta' +sql = "select tbname from devices limit 10"; +fprintf("Execute query: %s\n", sql); +tic; +stbmeta = select(conn, sql); +timeused = toc; +fprintf("\tTables in supertable 'devices': %t", stbmeta); +fprintf("\tQuery completed!\n\tNumber of rows retrieved: %d\n\tNumber of columns in each row: %d\n\tTime used: %g\n", height(stbmeta), width(stbmeta), timeused); + +%% Query a TDengine supertable, and stores the results in a MATLAB table object 'stb' +sql = "select ts, distance1 from devices"; +fprintf("Execute query: %s\n", sql); +tic; +stb = select(conn, sql); +timeused = toc; +fprintf("\tQuery completed!\n\tNumber of rows retrieved: %d\n\tNumber of columns in each row: %d\n\tTime used: %g\n", height(stb), width(stb), timeused); + +%% Query TDengine using cursors and specify the number of rows to fetch +sql = 'select * from device1'; +rowLimit = 5; +fprintf("Execute query: %s with row limit set to %d\n", sql, rowLimit); +tic; +% Get cursor +cur = exec(conn, sql); +% Fetch data +cur = fetch(cur, rowLimit); +data = cur.Data +timeused = toc; +fprintf("\tQuery completed!\n\tNumber of rows retrieved: %d\n\tNumber of columns in each row: %d\n\tTime used: %g\n", size(data, 1), size(data, 2), timeused); + +%% Query specific columns in a TDenigine table 'device1', and stores the results directly in a MATLAB cell array 'data' +sql = 'SELECT * FROM device1 order by ts asc'; +fprintf("Execute query: %s\n", sql); +tic; +data = fetch(conn, sql); +timeused = toc; +fprintf("\tQuery completed!\n\tNumber of rows retrieved: %d\n\tNumber of columns in each row: %d\n\tTime used: %g\n", size(data, 1), size(data, 2), timeused); +% Let's now convert the cell array 'data' into some matrices, and make a plot of column 'c1' again the timestamp 'ts' +ts = cell2mat(data(:,1)); +c1 = cell2mat(data(:,2)); + +%% Query aggregation results from a table +% TDengine is so powerful at aggregated computations. Let's calculate the max, mean, standard deviation and min values for every 10 minutes in the +% tb1's timeline, and then plot them together with all the data points in tb1 +sql = sprintf('SELECT max(measure1), avg(measure1), stddev(measure1), min(measure1) FROM device1 WHERE ts >= %d and ts <= %d interval(10m)', ts(1), ts(end)); +fprintf("Execute query: %s\n", sql); +tic; +c1_stats = fetch(conn, sql); +timeused = toc; +fprintf("\tQuery completed!\n\tNumber of rows retrieved: %d\n\tNumber of columns in each row: %d\n\tTime used: %g\n", size(c1_stats, 1), size(c1_stats, 2), timeused); +% Prepare data for plotting. +tsAsDate = datestr(ts/86400000 + datenum(1970,1,1), 'mm-dd HH:MM'); +c1_stats = cell2mat(c1_stats); +c1_stats_ts = c1_stats(:, 1); +c1_stats_max = c1_stats(:, 2); +c1_stats_mean = c1_stats(:, 3); +c1_stats_stddev = c1_stats(:, 4); +c1_stats_min = c1_stats(:, 5); +c1_stats_tsAsDate = datestr(c1_stats(:,1)/86400000 + datenum(1970,1,1), 'mm-dd HH:MM'); + +%% Now let's plot the data and associated statistical aggregation calculation in a figure. +fh = figure(1); +set(fh,'position',[50 50 1300 700]); +h1 = scatter(ts, c1, 5, 'c'); +hold on; +h2 = plot(c1_stats_ts + 300000, c1_stats_max, '-or', 'linewidth', 1); +hold on; +h3 = plot(c1_stats_ts + 300000, c1_stats_mean, '-xg', 'linewidth', 1); +hold on; +h4 = plot(c1_stats_ts + 300000, c1_stats_stddev, '-*y', 'linewidth', 1); +hold on; +h5 = plot(c1_stats_ts + 300000, c1_stats_min, '-+k', 'linewidth', 1); +xlabel('time'); +ylabel('measurement1'); +set(gca, 'xtick',[ts(1),ts(end/4),ts(2*end/4),ts(3*end/4),ts(end)]); +set(gca, 'xticklabel',{tsAsDate(1,:), tsAsDate(end/4,:),tsAsDate(2*end/4,:),tsAsDate(3*end/4,:),tsAsDate(end,:)}); +xlim([ts(1), ts(end)]); +legend([h1, h2, h3, h4, h5], 'data points', 'max per 10 mins', 'mean per 10 mins', 'stddev per 10 mins', 'min per 10 mins'); +title('Device Measurement Monitoring Demo'); +grid on; + +%% Insert data into TDengine using exec() +sql = 'insert into device1 (ts, distance1) values (now, -1)'; +fprintf("Execute query: %s\n", sql); +cur = exec(conn, sql) +sql = 'select * from device1 limit 1'; +fprintf("Execute query: %s\n", sql); +data = select(conn, sql) +conn.close; + +%% Insert data into TDengine using datainsert() +% this is currently not supported + +% colnames = {'ts','c1','c2','c3'}; +% dat = {'now' 99 99 99}; +% tbname = 'plane1'; +% datainsert(conn, tbname, colnames, dat); +% cur = exec(conn, 'select * from ' + tbname); +% cur = fetch(cur, 5); +% data = cur.Data + diff --git a/tests/examples/python/read_example.py b/tests/examples/python/read_example.py index 3276f296a2..a96115dfb2 100644 --- a/tests/examples/python/read_example.py +++ b/tests/examples/python/read_example.py @@ -1,83 +1,83 @@ -""" -This is the sample code for TDengine python2 client. -""" -import taos -import sys -import datetime -import random - -def exitProgram(conn): - conn.close() - sys.exit() - -if __name__ == '__main__': - start_time = datetime.datetime(2019, 7, 1) - time_interval = datetime.timedelta(seconds=60) - - # Connect to TDengine server. - # - # parameters: - # @host : TDengine server IP address - # @user : Username used to connect to TDengine server - # @password : Password - # @database : Database to use when connecting to TDengine server - # @config : Configuration directory - conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos") - - # Generate a cursor object to run SQL commands - c1 = conn.cursor() - - # Create a database named db - try: - c1.execute('create database db') - except Exception as err: - conn.close() - raise(err) - - # use database - try: - c1.execute('use db') - except Exception as err: - conn.close() - raise(err) - - - # create table - try: - c1.execute('create table if not exists t (ts timestamp, a int, b float, c binary(20))') - except Exception as err: - conn.close() - raise(err) - - # insert data - for i in range(10000): - try: - c1.execute("insert into t values ('%s', %d, %f, '%s')" % (start_time, random.randint(1,10), random.randint(1,10)/10.0, 'hello')) - except Exception as err: - conn.close() - raise(err) - start_time += time_interval - - # query data and return data in the form of list - try: - c1.execute('select * from db.t') - except Exception as err: - conn.close() - raise(err) - - # Column names are in c1.description list - cols = c1.description - # Use fetchall to fetch data in a list - data = c1.fetchall() - - try: - c1.execute('select * from db.t') - except Exception as err: - conn.close() - raise(err) - - # Use iterator to go through the retreived data - for col in c1: - print(col) - - conn.close() +""" +This is the sample code for TDengine python2 client. +""" +import taos +import sys +import datetime +import random + +def exitProgram(conn): + conn.close() + sys.exit() + +if __name__ == '__main__': + start_time = datetime.datetime(2019, 7, 1) + time_interval = datetime.timedelta(seconds=60) + + # Connect to TDengine server. + # + # parameters: + # @host : TDengine server IP address + # @user : Username used to connect to TDengine server + # @password : Password + # @database : Database to use when connecting to TDengine server + # @config : Configuration directory + conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos") + + # Generate a cursor object to run SQL commands + c1 = conn.cursor() + + # Create a database named db + try: + c1.execute('create database db') + except Exception as err: + conn.close() + raise(err) + + # use database + try: + c1.execute('use db') + except Exception as err: + conn.close() + raise(err) + + + # create table + try: + c1.execute('create table if not exists t (ts timestamp, a int, b float, c binary(20))') + except Exception as err: + conn.close() + raise(err) + + # insert data + for i in range(10000): + try: + c1.execute("insert into t values ('%s', %d, %f, '%s')" % (start_time, random.randint(1,10), random.randint(1,10)/10.0, 'hello')) + except Exception as err: + conn.close() + raise(err) + start_time += time_interval + + # query data and return data in the form of list + try: + c1.execute('select * from db.t') + except Exception as err: + conn.close() + raise(err) + + # Column names are in c1.description list + cols = c1.description + # Use fetchall to fetch data in a list + data = c1.fetchall() + + try: + c1.execute('select * from db.t') + except Exception as err: + conn.close() + raise(err) + + # Use iterator to go through the retreived data + for col in c1: + print(col) + + conn.close() -- GitLab